From a7c7cab00f08cabde3e41c594f1b5b7a7785320c Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 24 Aug 2026 18:41:31 -0600 Subject: [PATCH 01/73] Add optional Control discovery connector --- README.md | 2 + .../votingplugin/proxy/VotingPluginProxy.java | 37 ++ .../proxy/VotingPluginProxyConfig.java | 17 + .../proxy/bungee/BungeeConfig.java | 35 ++ .../proxy/bungee/VotingPluginBungee.java | 5 + .../proxy/control/ControlConnector.java | 500 ++++++++++++++++++ .../proxy/velocity/VelocityConfig.java | 35 ++ .../proxy/velocity/VotingPluginVelocity.java | 19 +- .../src/main/resources/bungeeconfig.yml | 11 + .../proxy/control/ControlConnectorTest.java | 165 ++++++ .../tests/VotingPluginProxyTestImpl.java | 5 + docs/control-connector.md | 90 ++++ 12 files changed, 914 insertions(+), 7 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java create mode 100644 docs/control-connector.md diff --git a/README.md b/README.md index 1a5130738a..45a3ae94a8 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Plugin on SpigotMC https://www.spigotmc.org/resources/votingplugin.15358/ +Development documentation: [optional VotingPlugin Control discovery connector](docs/control-connector.md). + ### Maven: diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 0a2fd7fdf0..4ecefef423 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -62,6 +62,7 @@ import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; +import com.bencodez.votingplugin.proxy.control.ControlConnector; import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyMethod; import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfiguration; @@ -161,6 +162,7 @@ public abstract class VotingPluginProxy { private final BackendPlayerPresenceTracker backendPlayerPresenceTracker = new BackendPlayerPresenceTracker(); private final Map pendingPresenceHandoffs = new HashMap<>(); private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); + private volatile ControlConnector controlConnector; public VotingPluginProxy() { enabled = true; @@ -1556,10 +1558,40 @@ public void onReceive(JsonEnvelope message) { loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, PRESENCE_MAINTENANCE_INTERVAL_SECONDS); } + startControlConnector(); debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); } + private synchronized void startControlConnector() { + stopControlConnector(); + if (!getConfig().getControlEnabled()) { + return; + } + try { + controlConnector = ControlConnector.create(this); + if (controlConnector != null) { + controlConnector.start(); + } + } catch (IOException | IllegalArgumentException e) { + controlConnector = null; + logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); + } + } + + private synchronized void stopControlConnector() { + ControlConnector connector = controlConnector; + controlConnector = null; + if (connector != null) { + connector.close(); + } + } + + public String getControlConnectorStatus() { + ControlConnector connector = controlConnector; + return connector == null ? "DISABLED" : connector.status().name(); + } + /** * Handles both the original login notification and extended presence logins. * Kept protected so transport-policy behavior can be regression tested without @@ -2262,6 +2294,7 @@ private void logInfo(String msg) { public abstract void logSevere(String message); public void onDisable() { + stopControlConnector(); getVoteCacheHandler().saveVoteCache(); if (getProxyMysqlMessenger() != null) { @@ -2408,6 +2441,7 @@ public void reload() { setCurrentVotePartyVotesRequired( getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); loadMultiProxySupport(); + startControlConnector(); } private void warnUnsupportedDedicatedVotingProxyMode() { @@ -2419,6 +2453,9 @@ private void warnUnsupportedDedicatedVotingProxyMode() { public abstract void runAsync(Runnable run); + /** Platform name used only for the transport-neutral Control discovery contract. */ + public abstract String getProxyPlatform(); + public abstract void runConsoleCommand(String command); public abstract void saveVoteCacheFile(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java index 27e631d85c..e183d3914e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java @@ -9,6 +9,23 @@ * Configuration interface for proxy server integration. */ public interface VotingPluginProxyConfig { + /** Whether the optional local VotingPlugin Control connector is enabled. */ + public boolean getControlEnabled(); + + /** Base HTTP(S) endpoint for VotingPlugin Control. */ + public String getControlEndpoint(); + + /** Stable enrolled identity; blank reuses ProxyServerName. */ + public String getControlNodeId(); + + /** Relative file in the plugin data folder containing the one-time enrollment credential. */ + public String getControlCredentialFile(); + + public int getControlHeartbeatSeconds(); + + public int getControlConnectTimeoutMillis(); + + public int getControlRequestTimeoutMillis(); /** * Gets whether proxy broadcast is enabled. diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index 4a74ae58f7..f715cd937c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -34,6 +34,41 @@ public BungeeConfig(VotingPluginBungee bungee) { this.bungee = bungee; } + @Override + public boolean getControlEnabled() { + return getData().getBoolean("Control.Enabled", false); + } + + @Override + public String getControlEndpoint() { + return getData().getString("Control.Endpoint", "http://127.0.0.1:8080"); + } + + @Override + public String getControlNodeId() { + return getData().getString("Control.NodeId", ""); + } + + @Override + public String getControlCredentialFile() { + return getData().getString("Control.CredentialFile", "control-credential.txt"); + } + + @Override + public int getControlHeartbeatSeconds() { + return getData().getInt("Control.HeartbeatSeconds", 30); + } + + @Override + public int getControlConnectTimeoutMillis() { + return getData().getInt("Control.ConnectTimeoutMillis", 3000); + } + + @Override + public int getControlRequestTimeoutMillis() { + return getData().getInt("Control.RequestTimeoutMillis", 5000); + } + /** * Converts a Configuration object to a Map. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index 40a229c5eb..4a42a1cdff 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -499,6 +499,11 @@ public VotingPluginProxyConfig getConfig() { } @Override + public String getProxyPlatform() { + return "BUNGEECORD"; + } + + @Override public String getCurrentPlayerServer(String player) { ProxiedPlayer p = getProxy().getPlayer(player); if (p != null && p.getServer() != null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java new file mode 100644 index 0000000000..51f2ab5bc7 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -0,0 +1,500 @@ +package com.bencodez.votingplugin.proxy.control; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HexFormat; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.LongSupplier; +import java.util.function.Supplier; +import java.util.regex.Pattern; + +import com.bencodez.votingplugin.proxy.VotingPluginProxy; +import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** + * Optional, failure-isolated Control discovery connector shared by BungeeCord and Velocity. + * It owns no vote-processing callbacks and never performs blocking I/O on a proxy event thread. + */ +public final class ControlConnector implements AutoCloseable { + static final int PROTOCOL_VERSION = 1; + static final int MAX_RESPONSE_BYTES = 64 * 1024; + private static final Pattern NODE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,63}"); + private static final Set CAPABILITIES = Set.of("presence.snapshot"); + private static final long MAX_BACKOFF_MILLIS = TimeUnit.MINUTES.toMillis(5); + + private final Settings settings; + private final ScheduledExecutorService scheduler; + private final Transport transport; + private final Supplier> snapshotSource; + private final Consumer logger; + private final UUID sessionId; + private final LongSupplier jitterSource; + private final AtomicBoolean inFlight = new AtomicBoolean(); + private volatile boolean closed; + private volatile boolean registered; + private volatile int failures; + private volatile long snapshotSequence; + private volatile ScheduledFuture scheduled; + private volatile CompletableFuture activeRequest; + private volatile Status status = Status.STARTING; + + public ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, + Supplier> snapshotSource, Consumer logger, UUID sessionId, + LongSupplier jitterSource) { + this.settings = Objects.requireNonNull(settings, "settings"); + this.scheduler = Objects.requireNonNull(scheduler, "scheduler"); + this.transport = Objects.requireNonNull(transport, "transport"); + this.snapshotSource = Objects.requireNonNull(snapshotSource, "snapshotSource"); + this.logger = Objects.requireNonNull(logger, "logger"); + this.sessionId = Objects.requireNonNull(sessionId, "sessionId"); + this.jitterSource = Objects.requireNonNull(jitterSource, "jitterSource"); + } + + /** Creates the production connector without reading a credential when the feature is disabled. */ + public static ControlConnector create(VotingPluginProxy proxy) throws IOException { + VotingPluginProxyConfig config = proxy.getConfig(); + if (!config.getControlEnabled()) { + return null; + } + String configuredNodeId = config.getControlNodeId(); + String nodeId = configuredNodeId == null || configuredNodeId.isBlank() + ? config.getProxyServerName() : configuredNodeId.trim(); + Path dataDirectory = proxy.getDataFolderPlugin().toPath().toAbsolutePath().normalize(); + String credentialName = config.getControlCredentialFile(); + if (credentialName == null || credentialName.isBlank()) { + throw new IllegalArgumentException("Control.CredentialFile must be set"); + } + Path credentialFile = dataDirectory.resolve(credentialName).normalize(); + if (!credentialFile.startsWith(dataDirectory)) { + throw new IllegalArgumentException("Control.CredentialFile must stay within the plugin data folder"); + } + String credential = readCredential(credentialFile); + String endpointValue = config.getControlEndpoint(); + if (endpointValue == null || endpointValue.isBlank()) { + throw new IllegalArgumentException("Control.Endpoint must be set"); + } + Settings settings = new Settings(nodeId, nodeId, proxy.getProxyPlatform(), proxy.getPluginVersion(), + URI.create(endpointValue.trim()), config.getControlHeartbeatSeconds(), + config.getControlConnectTimeoutMillis(), config.getControlRequestTimeoutMillis()); + HttpControlTransport transport = new HttpControlTransport(settings.endpoint(), credential, + settings.connectTimeoutMillis(), settings.requestTimeoutMillis()); + Supplier> snapshot = () -> { + List backends = new ArrayList<>(); + for (String server : proxy.getAllAvailableServers()) { + if (server == null || server.isBlank()) { + continue; + } + BackendPresenceStatus presence = proxy.getBackendPlayerPresenceTracker().getBackendStatus(server); + String displayName = server.trim(); + backends.add(new ObservedBackend(stableBackendId(displayName), + displayName.substring(0, Math.min(displayName.length(), 100)), presence != null, + presence != null && presence.isAvailable(), presence == null ? 0 : presence.getPlayerCount())); + } + backends.sort(Comparator.comparing(ObservedBackend::backendId)); + return List.copyOf(backends); + }; + return new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, + message -> proxy.log("[Control] " + message), UUID.randomUUID(), + () -> ThreadLocalRandom.current().nextLong()); + } + + public void start() { + if (closed) { + return; + } + status = Status.STARTING; + schedule(0); + } + + public Status status() { + return status; + } + + private void schedule(long delayMillis) { + if (closed) { + return; + } + try { + scheduled = scheduler.schedule(this::cycle, delayMillis, TimeUnit.MILLISECONDS); + } catch (java.util.concurrent.RejectedExecutionException ignored) { + if (!closed) { + status = Status.UNAVAILABLE; + } + } + } + + void cycle() { + if (closed || !inFlight.compareAndSet(false, true)) { + return; + } + Request first = registered ? heartbeatRequest() : registrationRequest(); + CompletableFuture primary = transport.send(first); + activeRequest = primary; + CompletableFuture operation = primary.thenCompose(response -> { + handlePrimaryResponse(response, !registered); + registered = true; + CompletableFuture presence = transport.send(presenceRequest()); + activeRequest = presence; + return presence.thenAccept(this::handlePresenceResponse); + }); + operation.whenComplete((ignored, failure) -> { + activeRequest = null; + inFlight.set(false); + if (closed) { + return; + } + if (failure == null) { + onSuccess(); + } else { + onFailure(unwrap(failure)); + } + }); + } + + private void handlePrimaryResponse(Response response, boolean registration) { + if (response.statusCode == 404 && !registration) { + registered = false; + throw new RegistryLostException(); + } + requireSuccess(response); + JsonObject body = parseObject(response.body); + if (registration) { + JsonElement protocol = body.getAsJsonObject("identity").get("protocolVersion"); + if (protocol == null || protocol.getAsInt() != PROTOCOL_VERSION) { + throw new ProtocolException(); + } + JsonArray accepted = body.getAsJsonObject("node").getAsJsonArray("acceptedCapabilities"); + if (accepted == null || !contains(accepted, "presence.snapshot")) { + throw new ProtocolException(); + } + } + } + + private void handlePresenceResponse(Response response) { + if (response.statusCode == 404) { + registered = false; + throw new RegistryLostException(); + } + requireSuccess(response); + JsonObject body = parseObject(response.body); + if (!body.has("applied") || !body.has("node")) { + throw new MalformedResponseException(); + } + } + + private static void requireSuccess(Response response) { + if (response.statusCode == 401 || response.statusCode == 403) { + throw new AuthenticationException(); + } + if (response.statusCode == 409) { + throw new ProtocolException(); + } + if (response.statusCode < 200 || response.statusCode >= 300) { + throw new UnavailableException(); + } + } + + private void onSuccess() { + boolean wasConnected = status == Status.CONNECTED; + failures = 0; + status = Status.CONNECTED; + if (!wasConnected) { + logger.accept("authenticated discovery connected"); + } + schedule(TimeUnit.SECONDS.toMillis(settings.heartbeatSeconds())); + } + + private void onFailure(Throwable failure) { + failures = Math.min(failures + 1, 30); + Status next; + String message; + if (failure instanceof AuthenticationException) { + next = Status.AUTHENTICATION_FAILED; + message = "authentication failed; re-enroll or rotate the configured credential"; + } else if (failure instanceof ProtocolException) { + next = Status.INCOMPATIBLE; + message = "protocol or required capabilities are incompatible"; + } else { + next = Status.UNAVAILABLE; + message = "unavailable; voting remains unaffected"; + } + if (status != next || failures == 1 || failures % 10 == 0) { + logger.accept(message); + } + status = next; + schedule(backoffMillis(failures, jitterSource.getAsLong())); + } + + private Request registrationRequest() { + JsonObject body = commonBody(); + body.addProperty("displayName", settings.displayName()); + body.addProperty("platform", settings.platform()); + body.addProperty("pluginVersion", settings.pluginVersion()); + addCapabilities(body); + return new Request("POST", "/api/v1/nodes/register", body.toString()); + } + + private Request heartbeatRequest() { + JsonObject body = sessionBody(); + addCapabilities(body); + return new Request("PUT", "/api/v1/nodes/" + settings.nodeId() + "/heartbeat", body.toString()); + } + + private Request presenceRequest() { + JsonObject body = sessionBody(); + body.addProperty("sequence", snapshotSequence++); + JsonArray backends = new JsonArray(); + List observed = snapshotSource.get(); + if (observed.size() > 4096) { + throw new IllegalStateException("Control backend snapshot exceeds 4096 entries"); + } + for (ObservedBackend backend : observed) { + JsonObject item = new JsonObject(); + item.addProperty("backendId", backend.backendId()); + item.addProperty("displayName", backend.displayName()); + item.addProperty("presenceKnown", backend.presenceKnown()); + item.addProperty("available", backend.available()); + item.addProperty("playerCount", backend.playerCount()); + backends.add(item); + } + body.add("backends", backends); + return new Request("PUT", "/api/v1/nodes/" + settings.nodeId() + "/presence", body.toString()); + } + + private JsonObject commonBody() { + JsonObject body = sessionBody(); + body.addProperty("nodeId", settings.nodeId()); + return body; + } + + private JsonObject sessionBody() { + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + body.addProperty("protocolVersion", PROTOCOL_VERSION); + return body; + } + + private static void addCapabilities(JsonObject body) { + JsonArray advertised = new JsonArray(); + CAPABILITIES.stream().sorted().forEach(advertised::add); + body.add("capabilities", advertised); + JsonArray required = new JsonArray(); + required.add("presence.snapshot"); + body.add("requiredCapabilities", required); + } + + static long backoffMillis(int failures, long jitterValue) { + int exponent = Math.max(0, Math.min(failures - 1, 8)); + long base = Math.min(1000L << exponent, MAX_BACKOFF_MILLIS); + long jitterBound = Math.max(1, base / 4); + return Math.min(MAX_BACKOFF_MILLIS, base + Math.floorMod(jitterValue, jitterBound)); + } + + static String stableBackendId(String serverName) { + String value = serverName == null ? "" : serverName.trim(); + if (NODE_ID.matcher(value).matches()) { + return value; + } + try { + byte[] digest = MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8)); + return "backend-" + HexFormat.of().formatHex(digest, 0, 16); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable"); + } + } + + @Override + public void close() { + closed = true; + status = Status.STOPPED; + ScheduledFuture scheduledRequest = scheduled; + if (scheduledRequest != null) { + scheduledRequest.cancel(false); + } + CompletableFuture request = activeRequest; + if (request != null) { + request.cancel(true); + } + transport.close(); + } + + private static String readCredential(Path path) throws IOException { + if (!Files.isRegularFile(path) || Files.size(path) > 512) { + throw new IOException("Control credential file is missing or invalid"); + } + String credential = Files.readString(path, StandardCharsets.UTF_8).trim(); + if (credential.isEmpty() || credential.length() > 512) { + throw new IOException("Control credential file is missing or invalid"); + } + return credential; + } + + private static JsonObject parseObject(String body) { + try { + JsonElement element = JsonParser.parseString(body); + if (!element.isJsonObject()) { + throw new MalformedResponseException(); + } + return element.getAsJsonObject(); + } catch (RuntimeException e) { + if (e instanceof ControlFailure) { + throw e; + } + throw new MalformedResponseException(); + } + } + + private static boolean contains(JsonArray array, String value) { + for (JsonElement element : array) { + if (element.isJsonPrimitive() && value.equals(element.getAsString())) { + return true; + } + } + return false; + } + + private static Throwable unwrap(Throwable failure) { + Throwable result = failure; + while (result instanceof CompletionException && result.getCause() != null) { + result = result.getCause(); + } + return result; + } + + public enum Status { + STARTING, CONNECTED, AUTHENTICATION_FAILED, INCOMPATIBLE, UNAVAILABLE, STOPPED + } + + public record Settings(String nodeId, String displayName, String platform, String pluginVersion, URI endpoint, + int heartbeatSeconds, int connectTimeoutMillis, int requestTimeoutMillis) { + public Settings { + if (nodeId == null || !NODE_ID.matcher(nodeId).matches()) { + throw new IllegalArgumentException("Control node ID is invalid"); + } + if (displayName == null || displayName.isBlank() || displayName.length() > 100) { + throw new IllegalArgumentException("Control display name is invalid"); + } + if (!Set.of("BUNGEECORD", "VELOCITY").contains(platform)) { + throw new IllegalArgumentException("Control platform is invalid"); + } + if (pluginVersion == null || pluginVersion.isBlank() || pluginVersion.length() > 40) { + throw new IllegalArgumentException("Control plugin version is invalid"); + } + Objects.requireNonNull(endpoint, "endpoint"); + if (!Set.of("http", "https").contains(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null + || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { + throw new IllegalArgumentException("Control endpoint must be an HTTP(S) origin without credentials or a path"); + } + if (heartbeatSeconds < 10 || heartbeatSeconds > 300) { + throw new IllegalArgumentException("Control heartbeat must be between 10 and 300 seconds"); + } + if (connectTimeoutMillis < 500 || connectTimeoutMillis > 30000 + || requestTimeoutMillis < 500 || requestTimeoutMillis > 30000) { + throw new IllegalArgumentException("Control timeouts must be between 500 and 30000 milliseconds"); + } + } + } + + public record ObservedBackend(String backendId, String displayName, boolean presenceKnown, + boolean available, int playerCount) { + public ObservedBackend { + if (backendId == null || !NODE_ID.matcher(backendId).matches()) { + throw new IllegalArgumentException("Control backend ID is invalid"); + } + if (displayName == null || displayName.isBlank() || displayName.length() > 100) { + throw new IllegalArgumentException("Control backend display name is invalid"); + } + if (playerCount < 0 || playerCount > 100000) { + throw new IllegalArgumentException("Control backend player count is invalid"); + } + } + } + + public record Request(String method, String path, String body) { } + public record Response(int statusCode, String body) { } + + public interface Transport extends AutoCloseable { + CompletableFuture send(Request request); + @Override default void close() { } + } + + static final class HttpControlTransport implements Transport { + private final URI endpoint; + private final String credential; + private final Duration requestTimeout; + private final HttpClient client; + + HttpControlTransport(URI endpoint, String credential, int connectTimeoutMillis, int requestTimeoutMillis) { + this.endpoint = endpoint; + this.credential = credential; + this.requestTimeout = Duration.ofMillis(requestTimeoutMillis); + this.client = HttpClient.newBuilder().connectTimeout(Duration.ofMillis(connectTimeoutMillis)) + .followRedirects(HttpClient.Redirect.NEVER).build(); + } + + @Override + public CompletableFuture send(Request request) { + HttpRequest httpRequest = HttpRequest.newBuilder(endpoint.resolve(request.path)) + .timeout(requestTimeout).header("Content-Type", "application/json") + .header("Authorization", "Bearer " + credential) + .method(request.method, HttpRequest.BodyPublishers.ofString(request.body, StandardCharsets.UTF_8)) + .build(); + return client.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofInputStream()) + .thenApply(response -> new Response(response.statusCode(), readBounded(response.body()))); + } + + private static String readBounded(InputStream input) { + try (input) { + byte[] bytes = input.readNBytes(MAX_RESPONSE_BYTES + 1); + if (bytes.length > MAX_RESPONSE_BYTES) { + throw new MalformedResponseException(); + } + return new String(bytes, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new CompletionException(new UnavailableException()); + } + } + } + + @SuppressWarnings("serial") + private abstract static class ControlFailure extends RuntimeException { } + @SuppressWarnings("serial") + private static final class AuthenticationException extends ControlFailure { } + @SuppressWarnings("serial") + private static final class ProtocolException extends ControlFailure { } + @SuppressWarnings("serial") + private static final class RegistryLostException extends ControlFailure { } + @SuppressWarnings("serial") + private static final class UnavailableException extends ControlFailure { } + @SuppressWarnings("serial") + private static final class MalformedResponseException extends ControlFailure { } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index ca745b995e..d5b9b0026a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -28,6 +28,41 @@ public VelocityConfig(File file) { super(file); } + @Override + public boolean getControlEnabled() { + return getBoolean(getNode("Control", "Enabled"), false); + } + + @Override + public String getControlEndpoint() { + return getString(getNode("Control", "Endpoint"), "http://127.0.0.1:8080"); + } + + @Override + public String getControlNodeId() { + return getString(getNode("Control", "NodeId"), ""); + } + + @Override + public String getControlCredentialFile() { + return getString(getNode("Control", "CredentialFile"), "control-credential.txt"); + } + + @Override + public int getControlHeartbeatSeconds() { + return getInt(getNode("Control", "HeartbeatSeconds"), 30); + } + + @Override + public int getControlConnectTimeoutMillis() { + return getInt(getNode("Control", "ConnectTimeoutMillis"), 3000); + } + + @Override + public int getControlRequestTimeoutMillis() { + return getInt(getNode("Control", "RequestTimeoutMillis"), 5000); + } + @Override public boolean getAllowUnJoined() { return getBoolean(getNode("AllowUnJoined"), false); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index a8ebda1084..693f4f48d3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -350,13 +350,7 @@ public void onProxyInitialization(ProxyInitializeEvent event) { CommandMeta meta = server.getCommandManager().metaBuilder("votingpluginproxy").aliases("vpp").build(); server.getCommandManager().register(meta, new VotingPluginVelocityCommand(this)); - // create runtime - votingPluginProxy = createProxyRuntime(); - - // full init - reloadAllInternal(true); - - // version + // Load the embedded version before the shared runtime starts optional integrations. try { getVersionFile(); if (versionFile != null) { @@ -368,6 +362,12 @@ public void onProxyInitialization(ProxyInitializeEvent event) { } catch (Exception ignored) { } + // create runtime + votingPluginProxy = createProxyRuntime(); + + // full init + reloadAllInternal(true); + // metrics (same as your original, shortened) Metrics metrics = metricsFactory.make(this, 11547); metrics.addCustomChart(new SimplePie("bungee_method", () -> getConfig().getBungeeMethod().toString())); @@ -582,6 +582,11 @@ public VotingPluginProxyConfig getConfig() { return config; } + @Override + public String getProxyPlatform() { + return "VELOCITY"; + } + @Override public String getCurrentPlayerServer(String player) { if (server.getPlayer(player).isPresent()) { diff --git a/VotingPlugin/src/main/resources/bungeeconfig.yml b/VotingPlugin/src/main/resources/bungeeconfig.yml index 321d835f6c..4427d64277 100644 --- a/VotingPlugin/src/main/resources/bungeeconfig.yml +++ b/VotingPlugin/src/main/resources/bungeeconfig.yml @@ -476,3 +476,14 @@ MultiProxyServers: secondproxy: Host: 0.0.0.0 Port: 1235 +# Optional local-first VotingPlugin Control discovery. Missing keys preserve the disabled default. +Control: + Enabled: false + Endpoint: 'http://127.0.0.1:8080' + # Blank reuses ProxyServerName. This must match the identity enrolled in Control. + NodeId: '' + # Relative to the VotingPlugin proxy data folder. Put only the enrollment credential in this file. + CredentialFile: 'control-credential.txt' + HeartbeatSeconds: 30 + ConnectTimeoutMillis: 3000 + RequestTimeoutMillis: 5000 diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java new file mode 100644 index 0000000000..a517a5bb90 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -0,0 +1,165 @@ +package com.bencodez.votingplugin.proxy.control; + +import com.bencodez.votingplugin.proxy.control.ControlConnector.ObservedBackend; +import com.bencodez.votingplugin.proxy.control.ControlConnector.Request; +import com.bencodez.votingplugin.proxy.control.ControlConnector.Response; +import com.bencodez.votingplugin.proxy.control.ControlConnector.Settings; +import com.bencodez.votingplugin.proxy.control.ControlConnector.Status; +import com.bencodez.votingplugin.proxy.control.ControlConnector.Transport; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import java.net.URI; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.*; + +class ControlConnectorTest { + private ScheduledExecutorService scheduler; + private FakeTransport transport; + private List logs; + private ControlConnector connector; + + @BeforeEach void setUp() { + scheduler = Executors.newSingleThreadScheduledExecutor(); + transport = new FakeTransport(); + logs = new ArrayList<>(); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(new ObservedBackend("lobby", "Lobby", true, true, 3)), logs::add, + UUID.fromString("00000000-0000-0000-0000-000000000001"), () -> 0L); + } + + @AfterEach void tearDown() { + connector.close(); + scheduler.shutdownNow(); + } + + @Test void registrationHeartbeatAndPresenceAreWireCompatibleAndReplacementSnapshotsAdvance() { + connector.cycle(); + assertEquals(Status.CONNECTED, connector.status()); + assertEquals(2, transport.requests.size()); + Request registration = transport.requests.get(0); + assertEquals("POST", registration.method()); + assertEquals("/api/v1/nodes/register", registration.path()); + JsonObject registrationJson = JsonParser.parseString(registration.body()).getAsJsonObject(); + assertEquals("proxy-a", registrationJson.get("nodeId").getAsString()); + assertEquals("VELOCITY", registrationJson.get("platform").getAsString()); + assertEquals(1, registrationJson.get("protocolVersion").getAsInt()); + assertEquals("presence.snapshot", registrationJson.getAsJsonArray("requiredCapabilities").get(0).getAsString()); + + JsonObject firstSnapshot = JsonParser.parseString(transport.requests.get(1).body()).getAsJsonObject(); + assertEquals(0, firstSnapshot.get("sequence").getAsLong()); + assertEquals("lobby", firstSnapshot.getAsJsonArray("backends").get(0).getAsJsonObject() + .get("backendId").getAsString()); + + connector.cycle(); + assertEquals("/api/v1/nodes/proxy-a/heartbeat", transport.requests.get(2).path()); + JsonObject secondSnapshot = JsonParser.parseString(transport.requests.get(3).body()).getAsJsonObject(); + assertEquals(1, secondSnapshot.get("sequence").getAsLong()); + } + + @Test void unavailableAuthenticationProtocolAndMalformedResponsesOnlyChangeConnectorState() { + transport.nextPrimary = new Response(401, "{\"error\":{}}"); + connector.cycle(); + assertEquals(Status.AUTHENTICATION_FAILED, connector.status()); + assertEquals(1, logs.size()); + + transport.nextPrimary = new Response(409, "{\"error\":{}}"); + connector.cycle(); + assertEquals(Status.INCOMPATIBLE, connector.status()); + + transport.nextPrimary = new Response(200, "not-json"); + connector.cycle(); + assertEquals(Status.UNAVAILABLE, connector.status()); + + transport.nextPrimary = new Response(503, "{\"error\":{}}"); + connector.cycle(); + assertEquals(Status.UNAVAILABLE, connector.status()); + } + + @Test void controlRegistryLossCausesReregistrationWithoutAffectingOtherCode() { + connector.cycle(); + transport.nextPrimary = new Response(404, "{\"error\":{}}"); + connector.cycle(); + assertEquals(Status.UNAVAILABLE, connector.status()); + connector.cycle(); + assertEquals("POST", transport.requests.get(3).method()); + assertEquals("/api/v1/nodes/register", transport.requests.get(3).path()); + } + + @Test void slowTransportDoesNotBlockCallerAndShutdownCancelsInFlightRequest() { + CompletableFuture stalled = new CompletableFuture<>(); + transport.stalled = stalled; + connector.cycle(); + assertFalse(stalled.isDone()); + assertEquals(Status.STARTING, connector.status()); + connector.close(); + assertTrue(stalled.isCancelled()); + assertEquals(Status.STOPPED, connector.status()); + } + + @Test void backoffIsBoundedExponentialAndJitteredWithoutSleeping() { + assertEquals(1000, ControlConnector.backoffMillis(1, 0)); + assertEquals(2000, ControlConnector.backoffMillis(2, 0)); + assertTrue(ControlConnector.backoffMillis(3, 999) >= 4000); + assertTrue(ControlConnector.backoffMillis(30, Long.MAX_VALUE) <= 300000); + assertTrue(ControlConnector.backoffMillis(30, Long.MIN_VALUE) > 0); + } + + @Test void backendIdentityPreservesSafeNamesAndDeterministicallyMapsOtherConfiguredNames() { + assertEquals("lobby-1", ControlConnector.stableBackendId("lobby-1")); + String mapped = ControlConnector.stableBackendId("Modded Lobby"); + assertTrue(mapped.matches("backend-[0-9a-f]{32}")); + assertEquals(mapped, ControlConnector.stableBackendId("Modded Lobby")); + assertNotEquals(mapped, ControlConnector.stableBackendId("Other Lobby")); + } + + @Test void settingsRejectCredentialsInUrlsUnsafePathsAndUnboundedTiming() { + assertThrows(IllegalArgumentException.class, () -> new Settings("proxy-a", "Proxy", "VELOCITY", "7.1.2", + URI.create("http://user:secret@localhost:8080"), 30, 3000, 5000)); + assertThrows(IllegalArgumentException.class, () -> new Settings("proxy-a", "Proxy", "VELOCITY", "7.1.2", + URI.create("http://localhost:8080/api"), 30, 3000, 5000)); + assertThrows(IllegalArgumentException.class, () -> new Settings("proxy-a", "Proxy", "VELOCITY", "7.1.2", + URI.create("http://localhost:8080"), 1, 3000, 5000)); + } + + private Settings settings() { + return new Settings("proxy-a", "Proxy A", "VELOCITY", "7.1.2", + URI.create("http://127.0.0.1:8080"), 30, 3000, 5000); + } + + private static final class FakeTransport implements Transport { + private final List requests = new ArrayList<>(); + private Response nextPrimary; + private CompletableFuture stalled; + + @Override + public CompletableFuture send(Request request) { + requests.add(request); + if (stalled != null) { + CompletableFuture result = stalled; + stalled = null; + return result; + } + if (!request.path().endsWith("/presence")) { + Response response = nextPrimary; + nextPrimary = null; + if (response != null) { + return CompletableFuture.completedFuture(response); + } + if ("POST".equals(request.method())) { + return CompletableFuture.completedFuture(new Response(201, + "{\"identity\":{\"protocolVersion\":1},\"node\":{\"acceptedCapabilities\":[\"presence.snapshot\"]}}")); + } + return CompletableFuture.completedFuture(new Response(200, "{\"node\":{}}")); + } + return CompletableFuture.completedFuture(new Response(200, "{\"applied\":true,\"node\":{}}")); + } + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 500c043708..3814ddbfcf 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -81,6 +81,11 @@ public String getPluginVersion() { return "1.0.0"; } + @Override + public String getProxyPlatform() { + return "VELOCITY"; + } + @Override public int getVoteCacheCurrentVotePartyVotes() { return 0; diff --git a/docs/control-connector.md b/docs/control-connector.md new file mode 100644 index 0000000000..fb3b390dd8 --- /dev/null +++ b/docs/control-connector.md @@ -0,0 +1,90 @@ +# VotingPlugin Control discovery connector + +This development milestone adds an optional, read-only connector from the BungeeCord and Velocity VotingPlugin proxy JAR +to the separate [VotingPlugin Control](https://github.com/BenCodez/VotingPlugin-Control) application. It discovers each +proxy and its eligible backend server names. It does not expose backend addresses or configuration and does not connect +directly from Bukkit backend nodes. + +## Failure isolation + +The connector is disabled by default. Missing `Control` keys therefore preserve existing installations exactly. When +enabled, startup performs only bounded local configuration/credential reads; network requests use Java's asynchronous HTTP +client outside proxy event-loop/main threads. There is at most one in-flight discovery cycle and no request queue. + +Control being stopped, slow, malformed, unauthorized, or protocol-incompatible changes only connector status and its +rate-limited `[Control]` log. Voting, proxy startup, joins, server switches, existing plugin messages, and shutdown do not +call into or wait on Control. Retries use exponential backoff from one second to five minutes with per-process jitter. +Shutdown cancels the scheduled cycle and active future without waiting indefinitely. + +## Enrollment and configuration + +1. Build/run Control and use its owner command to enroll the exact stable node ID: + + ```shell + java -jar votingplugin-control-0.1.0-SNAPSHOT-all.jar enroll proxy-a data + ``` + +2. On that proxy, create `plugins/VotingPlugin/control-credential.txt` (or the platform-equivalent VotingPlugin proxy data + directory) containing only the printed credential. Restrict filesystem access to the proxy service account. + +3. Configure `bungeeconfig.yml`: + + ```yaml + Control: + Enabled: true + Endpoint: 'http://127.0.0.1:8080' + NodeId: 'proxy-a' + CredentialFile: 'control-credential.txt' + HeartbeatSeconds: 30 + ConnectTimeoutMillis: 3000 + RequestTimeoutMillis: 5000 + ``` + +`NodeId` may be blank to reuse the existing `ProxyServerName`, but every simultaneously connected proxy must have a unique, +stable enrolled identity. Explicitly setting it is recommended. `CredentialFile` must resolve inside VotingPlugin's data +directory; credentials in URLs and parent-directory traversal are rejected. The credential is never logged or placed in +plugin messages. + +The endpoint must be an `http` or `https` origin without embedded credentials, query, fragment, or path. Plain HTTP is +suitable only for loopback or a trusted private network. Use HTTPS or a private authenticated tunnel when crossing an +untrusted network. Control authentication does not itself encrypt traffic. + +Timing bounds are intentional: + +- heartbeat: 10–300 seconds; +- connect/request timeout: 500–30,000 milliseconds; +- response body: at most 64 KiB; +- backend snapshot: at most 4096 entries. + +Do not raise these by patching around validation; correct the topology or connectivity problem instead. + +## Discovery semantics + +Both platforms use the same connector implementation and protocol version `1`. Each proxy process creates a new session +ID, registers, and sends full replacement backend snapshots with a monotonically increasing sequence. Registration and +retries are idempotent. A Control restart is detected by a 404 heartbeat/presence response and causes re-registration. + +Backend identity preserves VotingPlugin's existing eligible server name when it is already a protocol-safe identifier; +other configured names receive a deterministic SHA-256-derived ID while retaining a bounded safe display name. For transports that support VotingPlugin's backend +presence protocol, the existing `BackendPlayerPresenceTracker` supplies availability and player count. Where that protocol +has no authoritative observation, the backend remains discoverable with `presenceKnown: false`; the connector does not +invent an online state from a server address or arbitrary wall-clock timestamp. + +The connector advertises and requires `presence.snapshot`. Authentication failures, unavailable required capabilities, +and protocol mismatches back off only the connector. Current redacted diagnostic status is one of `DISABLED`, `STARTING`, +`CONNECTED`, `AUTHENTICATION_FAILED`, `INCOMPATIBLE`, `UNAVAILABLE`, or `STOPPED`. + +## Rotation, disabling, and troubleshooting + +- Rotate with Control's `enroll ` command, replace the credential file, then reload/restart VotingPlugin. +- Revoke immediately with Control's `revoke ` command. +- Disable by setting `Control.Enabled: false` and reloading/restarting; deleting all new keys also returns to the disabled + default. +- `AUTHENTICATION_FAILED`: confirm exact node-ID enrollment and replace a revoked/rotated credential. +- `INCOMPATIBLE`: the two JARs do not share protocol version/capability support; upgrade the older side. +- `UNAVAILABLE`: verify endpoint routing, Control health, TLS trust, and timeout settings. +- Backend listed with unknown presence: this is expected when the selected existing VotingPlugin transport does not provide + backend-presence observations. + +Configuration reads/writes, a WebUI, cloud relay, diagnostics downloads, automatic Control distribution, and remote support +remain separate later milestones. From b455be1ca26ef5c41ffcf83c490419fc65d3e105 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 24 Aug 2026 19:32:26 -0600 Subject: [PATCH 02/73] Add opt-in hosted Control distribution --- .../votingplugin/proxy/VotingPluginProxy.java | 39 +- .../proxy/VotingPluginProxyConfig.java | 47 ++ .../proxy/bungee/BungeeConfig.java | 55 ++ .../proxy/control/HostedControlManager.java | 611 ++++++++++++++++++ .../proxy/velocity/VelocityConfig.java | 55 ++ .../src/main/resources/bungeeconfig.yml | 18 + .../control/HostedControlManagerTest.java | 158 +++++ docs/control-connector.md | 62 +- 8 files changed, 1034 insertions(+), 11 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 4ecefef423..43c67b93b6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -63,6 +63,7 @@ import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; import com.bencodez.votingplugin.proxy.control.ControlConnector; +import com.bencodez.votingplugin.proxy.control.HostedControlManager; import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyMethod; import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfiguration; @@ -163,6 +164,7 @@ public abstract class VotingPluginProxy { private final Map pendingPresenceHandoffs = new HashMap<>(); private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); private volatile ControlConnector controlConnector; + private volatile HostedControlManager hostedControlManager; public VotingPluginProxy() { enabled = true; @@ -1558,13 +1560,24 @@ public void onReceive(JsonEnvelope message) { loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, PRESENCE_MAINTENANCE_INTERVAL_SECONDS); } - startControlConnector(); + startControlServices(); debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); } - private synchronized void startControlConnector() { - stopControlConnector(); + private synchronized void startControlServices() { + stopControlServices(true); + if (getConfig().getControlHostedEnabled()) { + try { + hostedControlManager = HostedControlManager.create(this); + if (hostedControlManager != null) { + hostedControlManager.start(); + } + } catch (IllegalArgumentException e) { + hostedControlManager = null; + logSevere("[Control Host] configuration is invalid; VotingPlugin remains unaffected"); + } + } if (!getConfig().getControlEnabled()) { return; } @@ -1579,12 +1592,21 @@ private synchronized void startControlConnector() { } } - private synchronized void stopControlConnector() { + private synchronized void stopControlServices(boolean waitForHosted) { ControlConnector connector = controlConnector; controlConnector = null; if (connector != null) { connector.close(); } + HostedControlManager manager = hostedControlManager; + hostedControlManager = null; + if (manager != null) { + if (waitForHosted) { + manager.closeAndWait(); + } else { + manager.close(); + } + } } public String getControlConnectorStatus() { @@ -1592,6 +1614,11 @@ public String getControlConnectorStatus() { return connector == null ? "DISABLED" : connector.status().name(); } + public String getHostedControlStatus() { + HostedControlManager manager = hostedControlManager; + return manager == null ? "DISABLED" : manager.status().name(); + } + /** * Handles both the original login notification and extended presence logins. * Kept protected so transport-policy behavior can be regression tested without @@ -2294,7 +2321,7 @@ private void logInfo(String msg) { public abstract void logSevere(String message); public void onDisable() { - stopControlConnector(); + stopControlServices(false); getVoteCacheHandler().saveVoteCache(); if (getProxyMysqlMessenger() != null) { @@ -2441,7 +2468,7 @@ public void reload() { setCurrentVotePartyVotesRequired( getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); loadMultiProxySupport(); - startControlConnector(); + startControlServices(); } private void warnUnsupportedDedicatedVotingProxyMode() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java index e183d3914e..ae71410c33 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java @@ -27,6 +27,53 @@ public interface VotingPluginProxyConfig { public int getControlRequestTimeoutMillis(); + /** Whether this proxy should provision and supervise a separate Control JVM. */ + default boolean getControlHostedEnabled() { + return false; + } + + /** Download the pinned artifact when it is not installed. */ + default boolean getControlHostedAutoDownload() { + return true; + } + + /** Replace an installed artifact when its digest differs from the configured pin. */ + default boolean getControlHostedAutoUpdate() { + return false; + } + + default String getControlHostedDownloadUrl() { + return ""; + } + + default String getControlHostedSha256() { + return ""; + } + + default String getControlHostedJarFile() { + return "control/votingplugin-control.jar"; + } + + default String getControlHostedDataDirectory() { + return "control/data"; + } + + default String getControlHostedHost() { + return "127.0.0.1"; + } + + default int getControlHostedPort() { + return 8080; + } + + default int getControlHostedStartupTimeoutSeconds() { + return 30; + } + + default int getControlHostedDownloadTimeoutSeconds() { + return 60; + } + /** * Gets whether proxy broadcast is enabled. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index f715cd937c..e4a9f57a3d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -69,6 +69,61 @@ public int getControlRequestTimeoutMillis() { return getData().getInt("Control.RequestTimeoutMillis", 5000); } + @Override + public boolean getControlHostedEnabled() { + return getData().getBoolean("Control.Hosted.Enabled", false); + } + + @Override + public boolean getControlHostedAutoDownload() { + return getData().getBoolean("Control.Hosted.AutoDownload", true); + } + + @Override + public boolean getControlHostedAutoUpdate() { + return getData().getBoolean("Control.Hosted.AutoUpdate", false); + } + + @Override + public String getControlHostedDownloadUrl() { + return getData().getString("Control.Hosted.DownloadUrl", ""); + } + + @Override + public String getControlHostedSha256() { + return getData().getString("Control.Hosted.Sha256", ""); + } + + @Override + public String getControlHostedJarFile() { + return getData().getString("Control.Hosted.JarFile", "control/votingplugin-control.jar"); + } + + @Override + public String getControlHostedDataDirectory() { + return getData().getString("Control.Hosted.DataDirectory", "control/data"); + } + + @Override + public String getControlHostedHost() { + return getData().getString("Control.Hosted.Host", "127.0.0.1"); + } + + @Override + public int getControlHostedPort() { + return getData().getInt("Control.Hosted.Port", 8080); + } + + @Override + public int getControlHostedStartupTimeoutSeconds() { + return getData().getInt("Control.Hosted.StartupTimeoutSeconds", 30); + } + + @Override + public int getControlHostedDownloadTimeoutSeconds() { + return getData().getInt("Control.Hosted.DownloadTimeoutSeconds", 60); + } + /** * Converts a Configuration object to a Map. * diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java new file mode 100644 index 0000000000..a213042728 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -0,0 +1,611 @@ +package com.bencodez.votingplugin.proxy.control; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.DigestInputStream; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Duration; +import java.util.HexFormat; +import java.util.Locale; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; +import java.util.function.LongSupplier; + +import com.bencodez.votingplugin.proxy.VotingPluginProxy; +import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** + * Opt-in provisioner for a separate VotingPlugin Control JVM. Downloads and + * process supervision never run in the proxy event loop or vote-processing path. + */ +public final class HostedControlManager implements AutoCloseable { + static final long MAX_DOWNLOAD_BYTES = 64L * 1024L * 1024L; + private static final int HEALTH_ATTEMPTS_PER_SECOND = 4; + private static final int EXPECTED_PROTOCOL_VERSION = 1; + + private final Settings settings; + private final ScheduledExecutorService executor; + private final ArtifactDownloader downloader; + private final ProcessLauncher launcher; + private final HealthProbe healthProbe; + private final Sleeper sleeper; + private final LongSupplier nanoTime; + private final Consumer logger; + private final AtomicBoolean workInProgress = new AtomicBoolean(); + private volatile boolean closed; + private volatile Status status = Status.STARTING; + private volatile ManagedProcess managedProcess; + private volatile Future activeTask; + private volatile int failures; + private volatile String quarantinedSha256; + + private HostedControlManager(Settings settings, Consumer logger) { + this(settings, daemonExecutor(), new JdkArtifactDownloader(), new JdkProcessLauncher(), + new JdkHealthProbe(), Thread::sleep, System::nanoTime, logger); + } + + HostedControlManager(Settings settings, ScheduledExecutorService executor, ArtifactDownloader downloader, + ProcessLauncher launcher, HealthProbe healthProbe, Sleeper sleeper, LongSupplier nanoTime, + Consumer logger) { + this.settings = Objects.requireNonNull(settings, "settings"); + this.executor = Objects.requireNonNull(executor, "executor"); + this.downloader = Objects.requireNonNull(downloader, "downloader"); + this.launcher = Objects.requireNonNull(launcher, "launcher"); + this.healthProbe = Objects.requireNonNull(healthProbe, "healthProbe"); + this.sleeper = Objects.requireNonNull(sleeper, "sleeper"); + this.nanoTime = Objects.requireNonNull(nanoTime, "nanoTime"); + this.logger = Objects.requireNonNull(logger, "logger"); + } + + public static HostedControlManager create(VotingPluginProxy proxy) { + VotingPluginProxyConfig config = proxy.getConfig(); + if (!config.getControlHostedEnabled()) { + return null; + } + Path root = proxy.getDataFolderPlugin().toPath().toAbsolutePath().normalize(); + Settings settings = new Settings(root, + resolveInside(root, config.getControlHostedJarFile(), "Control hosted JAR"), + resolveInside(root, config.getControlHostedDataDirectory(), "Control hosted data directory"), + config.getControlHostedAutoDownload(), config.getControlHostedAutoUpdate(), + parseDownloadUri(config.getControlHostedDownloadUrl()), config.getControlHostedSha256(), + config.getControlHostedHost(), config.getControlHostedPort(), + config.getControlHostedStartupTimeoutSeconds(), config.getControlHostedDownloadTimeoutSeconds()); + return new HostedControlManager(settings, message -> proxy.log("[Control Host] " + message)); + } + + public void start() { + if (closed || activeTask != null) { + return; + } + activeTask = executor.submit(this::runOnce); + } + + void runOnce() { + if (closed || !workInProgress.compareAndSet(false, true)) { + return; + } + try { + boolean updated = prepareArtifact(); + if (closed) { + return; + } + ManagedProcess process = launch(settings.jarFile()); + if (awaitHealthy(process)) { + failures = 0; + status = Status.RUNNING; + logger.accept("WebUI is available at " + settings.endpoint()); + monitor(process); + return; + } + stopProcess(process, true); + if (updated && Files.isRegularFile(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { + rollback(); + ManagedProcess previous = launch(settings.jarFile()); + if (awaitHealthy(previous)) { + failures = 0; + quarantinedSha256 = settings.sha256(); + status = Status.ROLLED_BACK; + logger.accept("The new Control release failed health checks; the previous release is running"); + monitor(previous); + return; + } + stopProcess(previous, true); + } + throw new IOException("Control did not become healthy"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + if (!closed) { + fail(); + } + } catch (Exception e) { + if (!closed) { + fail(); + } + } finally { + workInProgress.set(false); + } + } + + private boolean prepareArtifact() throws IOException, InterruptedException { + secureDirectory(settings.rootDirectory(), settings.jarFile().getParent()); + secureDirectory(settings.rootDirectory(), settings.dataDirectory()); + if (Files.isRegularFile(settings.jarFile(), LinkOption.NOFOLLOW_LINKS)) { + String installedSha256 = sha256(settings.jarFile()); + if (settings.sha256().equals(installedSha256) + || settings.sha256().equals(quarantinedSha256)) { + return false; + } + } + boolean installed = Files.exists(settings.jarFile(), LinkOption.NOFOLLOW_LINKS); + if (installed && !Files.isRegularFile(settings.jarFile(), LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Control artifact path is not a regular file"); + } + if ((installed && !settings.autoUpdate()) || (!installed && !settings.autoDownload())) { + throw new IOException("Control artifact is unavailable or does not match its pin"); + } + if (settings.downloadUri() == null) { + throw new IOException("Control download URL is not configured"); + } + + status = Status.DOWNLOADING; + Path staged = settings.jarFile().resolveSibling(settings.jarFile().getFileName() + ".staged-" + UUID.randomUUID()); + try { + downloader.download(settings.downloadUri(), staged, MAX_DOWNLOAD_BYTES, + Duration.ofSeconds(settings.downloadTimeoutSeconds())); + if (!settings.sha256().equals(sha256(staged))) { + throw new IOException("Downloaded Control artifact failed SHA-256 verification"); + } + activate(staged, installed); + return installed; + } finally { + Files.deleteIfExists(staged); + } + } + + private void activate(Path staged, boolean installed) throws IOException { + if (installed) { + atomicMove(settings.jarFile(), settings.previousFile(), true); + } + try { + atomicMove(staged, settings.jarFile(), false); + } catch (IOException e) { + if (installed && Files.exists(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { + atomicMove(settings.previousFile(), settings.jarFile(), true); + } + throw e; + } + } + + private void rollback() throws IOException { + if (Files.exists(settings.jarFile(), LinkOption.NOFOLLOW_LINKS)) { + atomicMove(settings.jarFile(), settings.failedFile(), true); + } + atomicMove(settings.previousFile(), settings.jarFile(), false); + } + + private ManagedProcess launch(Path artifact) throws IOException { + status = Status.STARTING_PROCESS; + ManagedProcess process = launcher.launch(settings, artifact); + managedProcess = process; + return process; + } + + private boolean awaitHealthy(ManagedProcess process) throws InterruptedException { + long timeoutNanos = TimeUnit.SECONDS.toNanos(settings.startupTimeoutSeconds()); + long started = nanoTime.getAsLong(); + long deadline = started + timeoutNanos; + if (deadline < started) { + deadline = Long.MAX_VALUE; + } + boolean firstAttempt = true; + while (!closed) { + if (!process.isAlive()) { + return false; + } + long now = nanoTime.getAsLong(); + if (!firstAttempt && now >= deadline) { + return false; + } + firstAttempt = false; + long remaining = Math.max(1L, deadline - now); + if (healthProbe.isHealthy(settings.endpoint(), + Duration.ofNanos(Math.min(TimeUnit.SECONDS.toNanos(2), remaining)))) { + return process.isAlive(); + } + if (nanoTime.getAsLong() >= deadline) { + return false; + } + sleeper.sleep(1000L / HEALTH_ATTEMPTS_PER_SECOND); + } + return false; + } + + private void monitor(ManagedProcess process) { + process.onExit().thenRun(() -> { + if (closed || managedProcess != process) { + return; + } + managedProcess = null; + status = Status.FAILED; + logger.accept("Control exited unexpectedly; a bounded restart will be attempted"); + scheduleRetry(); + }); + } + + private void fail() { + status = Status.FAILED; + logger.accept("Control could not be provisioned or started; VotingPlugin remains unaffected and will retry"); + scheduleRetry(); + } + + private void scheduleRetry() { + if (closed) { + return; + } + int attempt = Math.min(failures++, 6); + long delay = Math.min(300L, 5L << attempt); + try { + executor.schedule(this::runOnce, delay, TimeUnit.SECONDS); + } catch (RuntimeException ignored) { + // Executor shutdown raced with the failure callback. + } + } + + public Status status() { + return status; + } + + @Override + public void close() { + close(false); + } + + /** Used only by an explicit plugin reload to avoid racing the replacement bind. */ + public void closeAndWait() { + close(true); + } + + private void close(boolean waitForProcess) { + closed = true; + status = Status.STOPPED; + Future task = activeTask; + if (task != null) { + task.cancel(true); + } + ManagedProcess process = managedProcess; + managedProcess = null; + if (process != null) { + stopProcess(process, waitForProcess); + } + executor.shutdownNow(); + } + + private static void stopProcess(ManagedProcess process, boolean wait) { + process.destroy(); + if (wait) { + try { + if (!process.waitFor(3, TimeUnit.SECONDS)) { + process.destroyForcibly(); + } + } catch (InterruptedException e) { + process.destroyForcibly(); + Thread.currentThread().interrupt(); + } + } else { + process.onExit().orTimeout(3, TimeUnit.SECONDS).exceptionally(failure -> { + process.destroyForcibly(); + return null; + }); + } + } + + static String sha256(Path file) throws IOException { + try (InputStream input = Files.newInputStream(file); + DigestInputStream digest = new DigestInputStream(input, MessageDigest.getInstance("SHA-256"))) { + digest.transferTo(OutputStream.nullOutputStream()); + return HexFormat.of().formatHex(digest.getMessageDigest().digest()); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + private static void secureDirectory(Path root, Path directory) throws IOException { + Files.createDirectories(root); + Files.createDirectories(directory); + Path realRoot = root.toRealPath(); + Path realDirectory = directory.toRealPath(); + if (!realDirectory.startsWith(realRoot)) { + throw new IOException("Control path escapes the plugin data directory"); + } + } + + private static void atomicMove(Path source, Path target, boolean replace) throws IOException { + try { + if (replace) { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } else { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); + } + } catch (AtomicMoveNotSupportedException e) { + throw new IOException("Atomic Control artifact activation is unsupported", e); + } + } + + static Path resolveInside(Path root, String configured, String name) { + if (configured == null || configured.isBlank()) { + throw new IllegalArgumentException(name + " is not configured"); + } + Path relative = Path.of(configured.trim()); + if (relative.isAbsolute()) { + throw new IllegalArgumentException(name + " must be relative to the plugin data directory"); + } + Path resolved = root.resolve(relative).normalize(); + if (resolved.equals(root) || !resolved.startsWith(root)) { + throw new IllegalArgumentException(name + " escapes the plugin data directory"); + } + return resolved; + } + + static URI parseDownloadUri(String configured) { + if (configured == null || configured.isBlank()) { + return null; + } + URI uri; + try { + uri = URI.create(configured.trim()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Control download URL is invalid"); + } + if (!"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null || uri.getUserInfo() != null + || uri.getQuery() != null || uri.getFragment() != null || uri.getPath() == null + || uri.getPath().toLowerCase(Locale.ROOT).contains("/latest/")) { + throw new IllegalArgumentException("Control download URL must be a versioned HTTPS release asset"); + } + return uri; + } + + private static ScheduledExecutorService daemonExecutor() { + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1, runnable -> { + Thread thread = new Thread(runnable, "votingplugin-control-host"); + thread.setDaemon(true); + return thread; + }); + executor.setRemoveOnCancelPolicy(true); + return executor; + } + + public enum Status { + STARTING, DOWNLOADING, STARTING_PROCESS, RUNNING, ROLLED_BACK, FAILED, STOPPED + } + + record Settings(Path rootDirectory, Path jarFile, Path dataDirectory, boolean autoDownload, boolean autoUpdate, + URI downloadUri, String sha256, String host, int port, int startupTimeoutSeconds, + int downloadTimeoutSeconds) { + Settings { + rootDirectory = Objects.requireNonNull(rootDirectory, "rootDirectory").toAbsolutePath().normalize(); + jarFile = Objects.requireNonNull(jarFile, "jarFile").toAbsolutePath().normalize(); + dataDirectory = Objects.requireNonNull(dataDirectory, "dataDirectory").toAbsolutePath().normalize(); + if (!jarFile.startsWith(rootDirectory) || !dataDirectory.startsWith(rootDirectory) + || jarFile.equals(rootDirectory) || dataDirectory.equals(rootDirectory) + || dataDirectory.startsWith(jarFile)) { + throw new IllegalArgumentException("Control hosted paths are invalid"); + } + sha256 = sha256 == null ? "" : sha256.trim().toLowerCase(Locale.ROOT); + if (!sha256.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("Control hosted SHA-256 pin must contain 64 hexadecimal characters"); + } + if ((autoDownload || autoUpdate) && downloadUri == null) { + throw new IllegalArgumentException("Control hosted download URL is required"); + } + host = host == null ? "" : host.trim(); + if (host.isEmpty() || host.indexOf('\0') >= 0) { + throw new IllegalArgumentException("Control hosted host is invalid"); + } + if (port < 1 || port > 65535 || startupTimeoutSeconds < 1 || startupTimeoutSeconds > 120 + || downloadTimeoutSeconds < 5 || downloadTimeoutSeconds > 300) { + throw new IllegalArgumentException("Control hosted bounds are invalid"); + } + } + + Path previousFile() { + return jarFile.resolveSibling(jarFile.getFileName() + ".previous"); + } + + Path failedFile() { + return jarFile.resolveSibling(jarFile.getFileName() + ".failed"); + } + + URI endpoint() { + try { + return new URI("http", null, host, port, "/", null, null); + } catch (Exception e) { + throw new IllegalArgumentException("Control hosted endpoint is invalid"); + } + } + } + + interface ArtifactDownloader { + void download(URI source, Path target, long maximumBytes, Duration timeout) + throws IOException, InterruptedException; + } + + interface ProcessLauncher { + ManagedProcess launch(Settings settings, Path artifact) throws IOException; + } + + interface HealthProbe { + boolean isHealthy(URI endpoint, Duration timeout); + } + + interface Sleeper { + void sleep(long millis) throws InterruptedException; + } + + interface ManagedProcess { + boolean isAlive(); + void destroy(); + void destroyForcibly(); + boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException; + CompletableFuture onExit(); + } + + private static final class JdkArtifactDownloader implements ArtifactDownloader { + private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)) + .followRedirects(HttpClient.Redirect.NEVER).build(); + + @Override + public void download(URI source, Path target, long maximumBytes, Duration timeout) + throws IOException, InterruptedException { + URI current = source; + for (int redirects = 0; redirects <= 5; redirects++) { + HttpRequest request = HttpRequest.newBuilder(current).timeout(timeout) + .header("Accept", "application/java-archive, application/octet-stream") + .header("User-Agent", "VotingPlugin-Control-Host").GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + int status = response.statusCode(); + if (status >= 300 && status < 400) { + try (InputStream ignored = response.body()) { + String location = response.headers().firstValue("Location") + .orElseThrow(() -> new IOException("Control download redirect is invalid")); + current = current.resolve(location); + if (!"https".equalsIgnoreCase(current.getScheme()) || current.getHost() == null + || current.getUserInfo() != null) { + throw new IOException("Control download redirected outside HTTPS"); + } + continue; + } + } + if (status != 200) { + response.body().close(); + throw new IOException("Control download did not return HTTP 200"); + } + long declared = response.headers().firstValueAsLong("Content-Length").orElse(-1L); + if (declared > maximumBytes) { + response.body().close(); + throw new IOException("Control artifact exceeds the download limit"); + } + try (InputStream input = response.body(); OutputStream output = Files.newOutputStream(target, + StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + byte[] buffer = new byte[8192]; + long total = 0; + int count; + while ((count = input.read(buffer)) != -1) { + total += count; + if (total > maximumBytes) { + throw new IOException("Control artifact exceeds the download limit"); + } + output.write(buffer, 0, count); + } + } + return; + } + throw new IOException("Control download redirected too many times"); + } + } + + private static final class JdkProcessLauncher implements ProcessLauncher { + @Override + public ManagedProcess launch(Settings settings, Path artifact) throws IOException { + Path log = artifact.resolveSibling("control-output.log"); + if (Files.exists(log, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(log, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Control output path is not a regular file"); + } + if (Files.isRegularFile(log, LinkOption.NOFOLLOW_LINKS) && Files.size(log) > 1024L * 1024L) { + Files.move(log, log.resolveSibling("control-output.previous.log"), StandardCopyOption.REPLACE_EXISTING); + } + Path java = Path.of(System.getProperty("java.home"), "bin", isWindows() ? "java.exe" : "java"); + ProcessBuilder builder = new ProcessBuilder(java.toString(), "-jar", artifact.toString()); + builder.directory(artifact.getParent().toFile()); + builder.redirectErrorStream(true); + builder.redirectOutput(ProcessBuilder.Redirect.appendTo(log.toFile())); + var environment = builder.environment(); + environment.clear(); + for (String safe : Set.of("LANG", "LC_ALL", "SYSTEMROOT", "WINDIR", "TEMP", "TMP")) { + String value = System.getenv(safe); + if (value != null) { + environment.put(safe, value); + } + } + environment.put("CONTROL_HOST", settings.host()); + environment.put("CONTROL_PORT", Integer.toString(settings.port())); + environment.put("CONTROL_DATA_DIR", settings.dataDirectory().toString()); + return new JdkManagedProcess(builder.start()); + } + + private static boolean isWindows() { + return System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win"); + } + } + + private static final class JdkHealthProbe implements HealthProbe { + private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)) + .followRedirects(HttpClient.Redirect.NEVER).build(); + + @Override + public boolean isHealthy(URI endpoint, Duration timeout) { + try { + HttpRequest request = HttpRequest.newBuilder(endpoint.resolve("/api/v1/health")) + .timeout(timeout).header("Accept", "application/json").GET().build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream input = response.body()) { + byte[] bytes = input.readNBytes(64 * 1024 + 1); + if (response.statusCode() != 200 || bytes.length > 64 * 1024) { + return false; + } + JsonElement parsed = JsonParser.parseString(new String(bytes, java.nio.charset.StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) { + return false; + } + JsonObject object = parsed.getAsJsonObject(); + JsonObject identity = object.has("identity") && object.get("identity").isJsonObject() + ? object.getAsJsonObject("identity") : null; + return object.has("status") && "ok".equals(object.get("status").getAsString()) + && identity != null && identity.has("protocolVersion") + && identity.get("protocolVersion").getAsInt() == EXPECTED_PROTOCOL_VERSION; + } + } catch (Exception e) { + return false; + } + } + } + + private static final class JdkManagedProcess implements ManagedProcess { + private final Process process; + + private JdkManagedProcess(Process process) { + this.process = process; + } + + @Override public boolean isAlive() { return process.isAlive(); } + @Override public void destroy() { process.destroy(); } + @Override public void destroyForcibly() { process.destroyForcibly(); } + @Override public boolean waitFor(long timeout, TimeUnit unit) throws InterruptedException { + return process.waitFor(timeout, unit); + } + @Override public CompletableFuture onExit() { return process.onExit().thenApply(ignored -> null); } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index d5b9b0026a..d2d53b587a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -63,6 +63,61 @@ public int getControlRequestTimeoutMillis() { return getInt(getNode("Control", "RequestTimeoutMillis"), 5000); } + @Override + public boolean getControlHostedEnabled() { + return getBoolean(getNode("Control", "Hosted", "Enabled"), false); + } + + @Override + public boolean getControlHostedAutoDownload() { + return getBoolean(getNode("Control", "Hosted", "AutoDownload"), true); + } + + @Override + public boolean getControlHostedAutoUpdate() { + return getBoolean(getNode("Control", "Hosted", "AutoUpdate"), false); + } + + @Override + public String getControlHostedDownloadUrl() { + return getString(getNode("Control", "Hosted", "DownloadUrl"), ""); + } + + @Override + public String getControlHostedSha256() { + return getString(getNode("Control", "Hosted", "Sha256"), ""); + } + + @Override + public String getControlHostedJarFile() { + return getString(getNode("Control", "Hosted", "JarFile"), "control/votingplugin-control.jar"); + } + + @Override + public String getControlHostedDataDirectory() { + return getString(getNode("Control", "Hosted", "DataDirectory"), "control/data"); + } + + @Override + public String getControlHostedHost() { + return getString(getNode("Control", "Hosted", "Host"), "127.0.0.1"); + } + + @Override + public int getControlHostedPort() { + return getInt(getNode("Control", "Hosted", "Port"), 8080); + } + + @Override + public int getControlHostedStartupTimeoutSeconds() { + return getInt(getNode("Control", "Hosted", "StartupTimeoutSeconds"), 30); + } + + @Override + public int getControlHostedDownloadTimeoutSeconds() { + return getInt(getNode("Control", "Hosted", "DownloadTimeoutSeconds"), 60); + } + @Override public boolean getAllowUnJoined() { return getBoolean(getNode("AllowUnJoined"), false); diff --git a/VotingPlugin/src/main/resources/bungeeconfig.yml b/VotingPlugin/src/main/resources/bungeeconfig.yml index 4427d64277..220f8d8b41 100644 --- a/VotingPlugin/src/main/resources/bungeeconfig.yml +++ b/VotingPlugin/src/main/resources/bungeeconfig.yml @@ -487,3 +487,21 @@ Control: HeartbeatSeconds: 30 ConnectTimeoutMillis: 3000 RequestTimeoutMillis: 5000 + # Optionally let this proxy provision and supervise VotingPlugin Control as a separate JVM. + # The WebUI/API remains isolated from the proxy and vote processing. + Hosted: + Enabled: false + # Download a missing artifact. An exact SHA-256 pin is always required. + AutoDownload: true + # Disabled by default. When enabled, a new configured SHA-256 may replace the installed artifact. + AutoUpdate: false + # Use a versioned release asset URL; /latest/ URLs are rejected. + DownloadUrl: '' + Sha256: '' + # Both paths must remain relative to the VotingPlugin data folder. + JarFile: 'control/votingplugin-control.jar' + DataDirectory: 'control/data' + Host: '127.0.0.1' + Port: 8080 + StartupTimeoutSeconds: 30 + DownloadTimeoutSeconds: 60 diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java new file mode 100644 index 0000000000..efbd5d2d29 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -0,0 +1,158 @@ +package com.bencodez.votingplugin.proxy.control; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class HostedControlManagerTest { + @TempDir + Path directory; + + @Test + void versionPinsAndContainedPathsAreRequired() throws Exception { + Path root = directory.toAbsolutePath().normalize(); + assertThrows(IllegalArgumentException.class, + () -> HostedControlManager.resolveInside(root, "../outside.jar", "jar")); + assertThrows(IllegalArgumentException.class, + () -> HostedControlManager.parseDownloadUri( + "https://github.com/BenCodez/VotingPlugin-Control/releases/latest/download/control.jar")); + assertThrows(IllegalArgumentException.class, () -> settings(root.resolve("control.jar"), root.resolve("data"), + true, false, "not-a-digest", 30)); + } + + @Test + void missingPinnedArtifactIsStagedVerifiedAndStarted() throws Exception { + byte[] release = "signed-release-content".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control/control.jar"); + HostedControlManager.Settings settings = settings(jar, root.resolve("control/data"), true, false, + digest(release), 30); + FakeLauncher launcher = new FakeLauncher(); + AtomicInteger downloads = new AtomicInteger(); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager manager = new HostedControlManager(settings, executor, + (source, target, maximum, timeout) -> { + downloads.incrementAndGet(); + Files.write(target, release); + }, launcher, (endpoint, timeout) -> true, + millis -> { throw new AssertionError("healthy startup must not sleep"); }, System::nanoTime, + message -> { }); + + manager.runOnce(); + + assertEquals(HostedControlManager.Status.RUNNING, manager.status()); + assertEquals(1, downloads.get()); + assertEquals(List.of("signed-release-content"), launcher.launchedContents); + assertEquals("signed-release-content", Files.readString(jar)); + manager.close(); + assertTrue(launcher.processes.get(0).destroyed); + } + + @Test + void failedUpdateRollsBackAndStartsPreviousRelease() throws Exception { + byte[] previous = "previous-release".getBytes(StandardCharsets.UTF_8); + byte[] update = "bad-new-release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control/control.jar"); + Files.createDirectories(jar.getParent()); + Files.write(jar, previous); + HostedControlManager.Settings settings = settings(jar, root.resolve("control/data"), true, true, + digest(update), 1); + FakeLauncher launcher = new FakeLauncher(); + AtomicInteger healthChecks = new AtomicInteger(); + AtomicLong clock = new AtomicLong(); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager manager = new HostedControlManager(settings, executor, + (source, target, maximum, timeout) -> Files.write(target, update), launcher, + (endpoint, timeout) -> healthChecks.incrementAndGet() > 4, + millis -> clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(millis)), clock::get, message -> { }); + + manager.runOnce(); + + assertEquals(HostedControlManager.Status.ROLLED_BACK, manager.status()); + assertEquals(List.of("bad-new-release", "previous-release"), launcher.launchedContents); + assertEquals("previous-release", Files.readString(jar)); + assertEquals("bad-new-release", Files.readString(settings.failedFile())); + assertTrue(launcher.processes.get(0).destroyed); + manager.close(); + } + + @Test + void manualPinnedInstallNeedsNoDownloaderAndCloseIsNonBlocking() throws Exception { + byte[] release = "manual-release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control/control.jar"); + Files.createDirectories(jar.getParent()); + Files.write(jar, release); + HostedControlManager.Settings settings = new HostedControlManager.Settings(root, jar, + root.resolve("control/data"), false, false, null, digest(release), "127.0.0.1", 8080, 30, 60); + FakeLauncher launcher = new FakeLauncher(); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager manager = new HostedControlManager(settings, executor, + (source, target, maximum, timeout) -> { throw new AssertionError("manual install downloaded"); }, + launcher, (endpoint, timeout) -> true, millis -> { }, System::nanoTime, message -> { }); + + manager.runOnce(); + manager.close(); + + assertEquals(HostedControlManager.Status.STOPPED, manager.status()); + assertFalse(launcher.processes.get(0).isAlive()); + } + + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, + String sha256, int startupTimeout) { + Path root = directory.toAbsolutePath().normalize(); + return new HostedControlManager.Settings(root, jar, data, autoDownload, autoUpdate, + URI.create("https://github.com/BenCodez/VotingPlugin-Control/releases/download/v0.1.0/control.jar"), + sha256, "127.0.0.1", 8080, startupTimeout, 60); + } + + private String digest(byte[] bytes) throws Exception { + Path input = directory.resolve("digest-" + System.nanoTime()); + Files.write(input, bytes); + return HostedControlManager.sha256(input); + } + + private static final class FakeLauncher implements HostedControlManager.ProcessLauncher { + private final List launchedContents = new ArrayList<>(); + private final List processes = new ArrayList<>(); + + @Override + public HostedControlManager.ManagedProcess launch(HostedControlManager.Settings settings, Path artifact) + throws java.io.IOException { + launchedContents.add(Files.readString(artifact)); + FakeProcess process = new FakeProcess(); + processes.add(process); + return process; + } + } + + private static final class FakeProcess implements HostedControlManager.ManagedProcess { + private final CompletableFuture exit = new CompletableFuture<>(); + private volatile boolean alive = true; + private volatile boolean destroyed; + + @Override public boolean isAlive() { return alive; } + @Override public void destroy() { destroyed = true; alive = false; exit.complete(null); } + @Override public void destroyForcibly() { destroy(); } + @Override public boolean waitFor(long timeout, TimeUnit unit) { return !alive; } + @Override public CompletableFuture onExit() { return exit; } + } +} diff --git a/docs/control-connector.md b/docs/control-connector.md index fb3b390dd8..7b72d3f8f1 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -2,8 +2,9 @@ This development milestone adds an optional, read-only connector from the BungeeCord and Velocity VotingPlugin proxy JAR to the separate [VotingPlugin Control](https://github.com/BenCodez/VotingPlugin-Control) application. It discovers each -proxy and its eligible backend server names. It does not expose backend addresses or configuration and does not connect -directly from Bukkit backend nodes. +proxy and its eligible backend server names. Control includes the local WebUI. VotingPlugin may now explicitly opt in to +provisioning and supervising that application as a separate child JVM; it is never loaded into the proxy classloader. The +connector does not expose backend addresses or configuration and does not connect directly from Bukkit backend nodes. ## Failure isolation @@ -16,6 +17,56 @@ rate-limited `[Control]` log. Voting, proxy startup, joins, server switches, exi call into or wait on Control. Retries use exponential backoff from one second to five minutes with per-process jitter. Shutdown cancels the scheduled cycle and active future without waiting indefinitely. +The hosted-Control manager is independently disabled by default. Downloads, hashing, activation, health checks, process +startup, and restart backoff run on its own daemon worker. Hosting failure changes only its redacted status/log and never +blocks the proxy event loop or vote processing. Stopping VotingPlugin asks only the child process it created to terminate +and applies a bounded asynchronous force-stop fallback. + +## Optional hosted WebUI + +Use one hosted Control instance for the network. Other proxies should use the connector against that instance rather than +each starting a competing service. Hosting and discovery are separate switches: `Control.Hosted.Enabled` runs Control, +while `Control.Enabled` reports this proxy to it. + +Control releases publish `votingplugin-control.jar` and `votingplugin-control.jar.sha256`. Configure an immutable versioned +asset URL and copy its exact digest into `Sha256`; `/latest/` URLs, unpinned artifacts, plaintext download URLs, redirects +outside HTTPS, and paths outside VotingPlugin's data folder are rejected. + +```yaml +Control: + Enabled: true + Endpoint: 'http://127.0.0.1:8080' + NodeId: 'proxy-a' + CredentialFile: 'control-credential.txt' + Hosted: + Enabled: true + AutoDownload: true + AutoUpdate: false + DownloadUrl: 'https://github.com/BenCodez/VotingPlugin-Control/releases/download/v0.1.0/votingplugin-control.jar' + Sha256: '<64-character SHA-256 from the pinned release>' + JarFile: 'control/votingplugin-control.jar' + DataDirectory: 'control/data' + Host: '127.0.0.1' + Port: 8080 + StartupTimeoutSeconds: 30 + DownloadTimeoutSeconds: 60 +``` + +On first start, the artifact is downloaded to a unique staging file with a 64 MiB hard limit, verified, and atomically +activated before launch with the same Java runtime as the proxy. An existing matching manual installation is launched +without downloading. `AutoUpdate` is false by default; when explicitly enabled and the configured digest changes, the +current release is retained as `.previous`. A failed process/protocol health check atomically restores and starts that +previous release, retaining the failed candidate as `.failed`. Unexpected exits use bounded restart backoff. + +The child receives only its bind host, port, and contained data directory. It does not receive VotingPlugin configuration +or credentials in release metadata. Process output is written beside the hosted JAR and rotated at 1 MiB. The WebUI is +then available at `http://127.0.0.1:8080/` and asks for Control's separate admin token. To expose a non-loopback listener, +create the admin token in `Control.Hosted.DataDirectory` first and use HTTPS or a private tunnel; Control otherwise refuses +the bind. + +Manual installation remains supported: put a Control JAR at `JarFile`, configure its exact SHA-256, set `AutoDownload` and +`AutoUpdate` false, and enable hosting. VotingPlugin will verify and supervise it without network access. + ## Enrollment and configuration 1. Build/run Control and use its owner command to enroll the exact stable node ID: @@ -79,12 +130,13 @@ and protocol mismatches back off only the connector. Current redacted diagnostic - Rotate with Control's `enroll ` command, replace the credential file, then reload/restart VotingPlugin. - Revoke immediately with Control's `revoke ` command. - Disable by setting `Control.Enabled: false` and reloading/restarting; deleting all new keys also returns to the disabled - default. +default. - `AUTHENTICATION_FAILED`: confirm exact node-ID enrollment and replace a revoked/rotated credential. - `INCOMPATIBLE`: the two JARs do not share protocol version/capability support; upgrade the older side. - `UNAVAILABLE`: verify endpoint routing, Control health, TLS trust, and timeout settings. - Backend listed with unknown presence: this is expected when the selected existing VotingPlugin transport does not provide backend-presence observations. -Configuration reads/writes, a WebUI, cloud relay, diagnostics downloads, automatic Control distribution, and remote support -remain separate later milestones. +Configuration reads/writes, topology persistence, cloud relay, diagnostics downloads, signed remote release manifests, +and remote support remain later milestones. Hosted downloads currently require the SHA-256 trust pin to be supplied in +local configuration; VotingPlugin never trusts a remotely fetched checksum by itself. From 276f1375248e025e516607913a28163b58727b27 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 24 Aug 2026 19:37:25 -0600 Subject: [PATCH 03/73] Test hosted Control digest rejection --- .../control/HostedControlManagerTest.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index efbd5d2d29..34edc43199 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -116,6 +116,28 @@ void manualPinnedInstallNeedsNoDownloaderAndCloseIsNonBlocking() throws Exceptio assertFalse(launcher.processes.get(0).isAlive()); } + @Test + void digestMismatchNeverActivatesOrLaunchesDownloadedCode() throws Exception { + byte[] expected = "expected-release".getBytes(StandardCharsets.UTF_8); + byte[] untrusted = "different-content".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control/control.jar"); + HostedControlManager.Settings settings = settings(jar, root.resolve("control/data"), true, false, + digest(expected), 30); + FakeLauncher launcher = new FakeLauncher(); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager manager = new HostedControlManager(settings, executor, + (source, target, maximum, timeout) -> Files.write(target, untrusted), launcher, + (endpoint, timeout) -> true, millis -> { }, System::nanoTime, message -> { }); + + manager.runOnce(); + + assertEquals(HostedControlManager.Status.FAILED, manager.status()); + assertFalse(Files.exists(jar)); + assertTrue(launcher.processes.isEmpty()); + manager.close(); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); From 4f4a49b5e578bde77127a5078ff8634c6e2909ba Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 24 Aug 2026 20:10:42 -0600 Subject: [PATCH 04/73] Add typed Control configuration operations --- .../proxy/VotingPluginProxyConfig.java | 39 ++++- .../proxy/bungee/BungeeConfig.java | 48 +++++- .../proxy/control/ControlConnector.java | 145 +++++++++++++++++- .../proxy/control/HostedControlManager.java | 9 +- .../control/ProxyRoutingConfiguration.java | 48 ++++++ .../ProxyRoutingConfigurationService.java | 62 ++++++++ .../proxy/velocity/VelocityConfig.java | 45 ++++++ .../proxy/control/ControlConnectorTest.java | 29 +++- .../control/HostedControlManagerTest.java | 4 + .../ProxyRoutingConfigurationTest.java | 24 +++ docs/control-connector.md | 22 ++- 11 files changed, 444 insertions(+), 31 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfiguration.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java index ae71410c33..ed1442da6c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java @@ -1,5 +1,6 @@ package com.bencodez.votingplugin.proxy; +import java.io.IOException; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -9,23 +10,47 @@ * Configuration interface for proxy server integration. */ public interface VotingPluginProxyConfig { + /** Atomically persists the small non-secret configuration domain exposed by Control. */ + default void persistControlProxyRouting(boolean sendVotesToAllServers, List blockedServers) + throws IOException { + throw new UnsupportedOperationException("Control configuration writes are unavailable"); + } + + /** Restores the most recent Control-created backup. */ + default void rollbackControlProxyRouting() throws IOException { + throw new UnsupportedOperationException("Control configuration rollback is unavailable"); + } /** Whether the optional local VotingPlugin Control connector is enabled. */ - public boolean getControlEnabled(); + default boolean getControlEnabled() { + return false; + } /** Base HTTP(S) endpoint for VotingPlugin Control. */ - public String getControlEndpoint(); + default String getControlEndpoint() { + return "http://127.0.0.1:8080"; + } /** Stable enrolled identity; blank reuses ProxyServerName. */ - public String getControlNodeId(); + default String getControlNodeId() { + return ""; + } /** Relative file in the plugin data folder containing the one-time enrollment credential. */ - public String getControlCredentialFile(); + default String getControlCredentialFile() { + return "control-credential.txt"; + } - public int getControlHeartbeatSeconds(); + default int getControlHeartbeatSeconds() { + return 30; + } - public int getControlConnectTimeoutMillis(); + default int getControlConnectTimeoutMillis() { + return 3000; + } - public int getControlRequestTimeoutMillis(); + default int getControlRequestTimeoutMillis() { + return 5000; + } /** Whether this proxy should provision and supervise a separate Control JVM. */ default boolean getControlHostedEnabled() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index e4a9f57a3d..8ebd3b1496 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -3,7 +3,9 @@ import java.io.File; import java.io.IOException; import java.io.InputStream; -import java.nio.file.Files; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -20,7 +22,7 @@ /** * Configuration implementation for Bungee proxy server. */ -public class BungeeConfig implements VotingPluginProxyConfig { +public class BungeeConfig implements VotingPluginProxyConfig { private VotingPluginBungee bungee; @Getter private Configuration data; @@ -498,14 +500,52 @@ public void load() { } } - public void save() { + public void save() { try { ConfigurationProvider.getProvider(YamlConfiguration.class).save(data, new File(bungee.getDataFolder(), "bungeeconfig.yml")); } catch (IOException e) { e.printStackTrace(); } - } + } + + @Override + public synchronized void persistControlProxyRouting(boolean sendVotesToAllServers, List blockedServers) + throws IOException { + Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); + Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-stage"); + Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); + Configuration candidate = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); + candidate.set("SendVotesToAllServers", sendVotesToAllServers); + candidate.set("BlockedServers", List.copyOf(blockedServers)); + try { + ConfigurationProvider.getProvider(YamlConfiguration.class).save(candidate, stage.toFile()); + Files.copy(target, backup, StandardCopyOption.REPLACE_EXISTING); + atomicReplace(stage, target); + data = candidate; + } finally { + Files.deleteIfExists(stage); + } + } + + @Override + public synchronized void rollbackControlProxyRouting() throws IOException { + Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); + Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); + if (!Files.isRegularFile(backup)) throw new IOException("Control backup is unavailable"); + Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); + Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); + atomicReplace(stage, target); + data = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); + } + + private static void atomicReplace(Path source, Path target) throws IOException { + try { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (java.nio.file.AtomicMoveNotSupportedException e) { + throw new IOException("Atomic Control configuration activation is unsupported", e); + } + } @Override public String getPluginMessageChannel() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 51f2ab5bc7..d1ea86aa1d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -16,6 +16,8 @@ import java.util.Comparator; import java.util.HexFormat; import java.util.List; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.UUID; @@ -47,7 +49,8 @@ public final class ControlConnector implements AutoCloseable { static final int PROTOCOL_VERSION = 1; static final int MAX_RESPONSE_BYTES = 64 * 1024; private static final Pattern NODE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,63}"); - private static final Set CAPABILITIES = Set.of("presence.snapshot"); + private static final Set BASE_CAPABILITIES = Set.of("presence.snapshot"); + private static final String CONFIGURATION_CAPABILITY = "config.proxy-routing.v1"; private static final long MAX_BACKOFF_MILLIS = TimeUnit.MINUTES.toMillis(5); private final Settings settings; @@ -57,6 +60,8 @@ public final class ControlConnector implements AutoCloseable { private final Consumer logger; private final UUID sessionId; private final LongSupplier jitterSource; + private final ProxyRoutingConfigurationService configurationService; + private final Map completedTasks = new LinkedHashMap<>(); private final AtomicBoolean inFlight = new AtomicBoolean(); private volatile boolean closed; private volatile boolean registered; @@ -69,6 +74,12 @@ public final class ControlConnector implements AutoCloseable { public ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, Supplier> snapshotSource, Consumer logger, UUID sessionId, LongSupplier jitterSource) { + this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, null); + } + + ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, + Supplier> snapshotSource, Consumer logger, UUID sessionId, + LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService) { this.settings = Objects.requireNonNull(settings, "settings"); this.scheduler = Objects.requireNonNull(scheduler, "scheduler"); this.transport = Objects.requireNonNull(transport, "transport"); @@ -76,6 +87,7 @@ public ControlConnector(Settings settings, ScheduledExecutorService scheduler, T this.logger = Objects.requireNonNull(logger, "logger"); this.sessionId = Objects.requireNonNull(sessionId, "sessionId"); this.jitterSource = Objects.requireNonNull(jitterSource, "jitterSource"); + this.configurationService = configurationService; } /** Creates the production connector without reading a credential when the feature is disabled. */ @@ -123,7 +135,7 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio }; return new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, message -> proxy.log("[Control] " + message), UUID.randomUUID(), - () -> ThreadLocalRandom.current().nextLong()); + () -> ThreadLocalRandom.current().nextLong(), new ProxyRoutingConfigurationService(proxy)); } public void start() { @@ -156,14 +168,26 @@ void cycle() { return; } Request first = registered ? heartbeatRequest() : registrationRequest(); - CompletableFuture primary = transport.send(first); + CompletableFuture primary; + try { + primary = transport.send(first); + } catch (RuntimeException failure) { + inFlight.set(false); + onFailure(failure); + return; + } activeRequest = primary; CompletableFuture operation = primary.thenCompose(response -> { handlePrimaryResponse(response, !registered); registered = true; CompletableFuture presence = transport.send(presenceRequest()); activeRequest = presence; - return presence.thenAccept(this::handlePresenceResponse); + return presence.thenCompose(responseBody -> { + handlePresenceResponse(responseBody); + CompletableFuture claim = transport.send(claimRequest()); + activeRequest = claim; + return claim.thenCompose(this::handleClaimResponse); + }); }); operation.whenComplete((ignored, failure) -> { activeRequest = null; @@ -289,6 +313,103 @@ private Request presenceRequest() { return new Request("PUT", "/api/v1/nodes/" + settings.nodeId() + "/presence", body.toString()); } + private Request claimRequest() { + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + return new Request("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations", body.toString()); + } + + private CompletableFuture handleClaimResponse(Response response) { + if (response.statusCode == 204) return CompletableFuture.completedFuture(null); + if (response.statusCode == 404) { + registered = false; + throw new RegistryLostException(); + } + requireSuccess(response); + JsonObject task = parseObject(response.body); + UUID operationId = UUID.fromString(requireString(task, "operationId")); + TaskResult result; + synchronized (completedTasks) { + result = completedTasks.get(operationId); + } + if (result == null) { + result = executeTask(task); + synchronized (completedTasks) { + completedTasks.put(operationId, result); + while (completedTasks.size() > 128) completedTasks.remove(completedTasks.keySet().iterator().next()); + } + } + CompletableFuture submitted = transport.send(resultRequest(operationId, result)); + activeRequest = submitted; + return submitted.thenAccept(ControlConnector::requireSuccess); + } + + private TaskResult executeTask(JsonObject task) { + if (configurationService == null) return TaskResult.failure("UNSUPPORTED", "Configuration control is unavailable"); + String type = requireString(task, "type"); + try { + ProxyRoutingConfiguration current = configurationService.read(); + if ("READ".equals(type)) return TaskResult.success(current.revision(), current, List.of(), false); + ProxyRoutingConfiguration proposal = parseConfiguration(task.getAsJsonObject("configuration")); + configurationService.validate(proposal); + List changes = proposal.changesFrom(current); + if ("PREVIEW".equals(type)) return TaskResult.success(current.revision(), current, changes, false); + if (!"APPLY".equals(type)) return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); + configurationService.apply(proposal, requireString(task, "expectedRevision")); + ProxyRoutingConfiguration applied = configurationService.read(); + return TaskResult.success(applied.revision(), applied, changes, true); + } catch (ProxyRoutingConfigurationService.StaleRevisionException e) { + return TaskResult.failure("STALE_REVISION", "Configuration changed after preview"); + } catch (ProxyRoutingConfigurationService.ApplyFailureException e) { + return new TaskResult(false, "RELOAD_FAILED", "Reload failed after persistence", null, null, List.of(), + false, e.rolledBack()); + } catch (IllegalArgumentException e) { + return TaskResult.failure("VALIDATION_ERROR", e.getMessage()); + } catch (IOException | RuntimeException e) { + return TaskResult.failure("APPLY_FAILED", "Configuration operation failed"); + } + } + + private static ProxyRoutingConfiguration parseConfiguration(JsonObject body) { + if (body == null || !body.has("sendVotesToAllServers") || !body.has("blockedServers")) { + throw new IllegalArgumentException("configuration is incomplete"); + } + List blocked = new ArrayList<>(); + body.getAsJsonArray("blockedServers").forEach(value -> blocked.add(value.getAsString())); + return new ProxyRoutingConfiguration(body.get("sendVotesToAllServers").getAsBoolean(), blocked); + } + + private Request resultRequest(UUID operationId, TaskResult result) { + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + body.addProperty("success", result.success); + body.addProperty("code", result.code); + body.addProperty("message", result.message); + if (result.revision != null) body.addProperty("revision", result.revision); + if (result.configuration != null) body.add("configuration", configurationJson(result.configuration)); + JsonArray changes = new JsonArray(); + result.changes.forEach(changes::add); + body.add("changes", changes); + body.addProperty("reloaded", result.reloaded); + body.addProperty("rolledBack", result.rolledBack); + return new Request("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + + "/result", body.toString()); + } + + private static JsonObject configurationJson(ProxyRoutingConfiguration configuration) { + JsonObject body = new JsonObject(); + body.addProperty("sendVotesToAllServers", configuration.sendVotesToAllServers()); + JsonArray blocked = new JsonArray(); + configuration.blockedServers().forEach(blocked::add); + body.add("blockedServers", blocked); + return body; + } + + private static String requireString(JsonObject body, String name) { + if (body == null || !body.has(name) || !body.get(name).isJsonPrimitive()) throw new MalformedResponseException(); + return body.get(name).getAsString(); + } + private JsonObject commonBody() { JsonObject body = sessionBody(); body.addProperty("nodeId", settings.nodeId()); @@ -302,9 +423,10 @@ private JsonObject sessionBody() { return body; } - private static void addCapabilities(JsonObject body) { + private void addCapabilities(JsonObject body) { JsonArray advertised = new JsonArray(); - CAPABILITIES.stream().sorted().forEach(advertised::add); + BASE_CAPABILITIES.stream().sorted().forEach(advertised::add); + if (configurationService != null) advertised.add(CONFIGURATION_CAPABILITY); body.add("capabilities", advertised); JsonArray required = new JsonArray(); required.add("presence.snapshot"); @@ -442,6 +564,17 @@ public record ObservedBackend(String backendId, String displayName, boolean pres public record Request(String method, String path, String body) { } public record Response(int statusCode, String body) { } + private record TaskResult(boolean success, String code, String message, String revision, + ProxyRoutingConfiguration configuration, List changes, boolean reloaded, boolean rolledBack) { + private static TaskResult success(String revision, ProxyRoutingConfiguration configuration, + List changes, boolean reloaded) { + return new TaskResult(true, "OK", "Operation completed", revision, configuration, changes, reloaded, false); + } + private static TaskResult failure(String code, String message) { + return new TaskResult(false, code, message == null ? "Operation failed" : message, null, null, List.of(), false, false); + } + } + public interface Transport extends AutoCloseable { CompletableFuture send(Request request); @Override default void close() { } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index a213042728..c068a1b794 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -61,6 +61,7 @@ public final class HostedControlManager implements AutoCloseable { private volatile Future activeTask; private volatile int failures; private volatile String quarantinedSha256; + private volatile String trustedRollbackSha256; private HostedControlManager(Settings settings, Consumer logger) { this(settings, daemonExecutor(), new JdkArtifactDownloader(), new JdkProcessLauncher(), @@ -154,10 +155,14 @@ private boolean prepareArtifact() throws IOException, InterruptedException { secureDirectory(settings.rootDirectory(), settings.dataDirectory()); if (Files.isRegularFile(settings.jarFile(), LinkOption.NOFOLLOW_LINKS)) { String installedSha256 = sha256(settings.jarFile()); - if (settings.sha256().equals(installedSha256) - || settings.sha256().equals(quarantinedSha256)) { + if (settings.sha256().equals(installedSha256)) { return false; } + if (settings.sha256().equals(quarantinedSha256)) { + if (installedSha256.equals(trustedRollbackSha256)) return false; + throw new IOException("Rolled-back Control artifact no longer matches its trusted digest"); + } + trustedRollbackSha256 = installedSha256; } boolean installed = Files.exists(settings.jarFile(), LinkOption.NOFOLLOW_LINKS); if (installed && !Files.isRegularFile(settings.jarFile(), LinkOption.NOFOLLOW_LINKS)) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfiguration.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfiguration.java new file mode 100644 index 0000000000..87d4758897 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfiguration.java @@ -0,0 +1,48 @@ +package com.bencodez.votingplugin.proxy.control; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Locale; +import java.util.Set; + +/** Typed, non-secret configuration domain supported by Control. */ +public record ProxyRoutingConfiguration(boolean sendVotesToAllServers, List blockedServers) { + public ProxyRoutingConfiguration { + blockedServers = blockedServers == null ? List.of() : blockedServers.stream().map(String::trim).toList(); + if (blockedServers.size() > 256) throw new IllegalArgumentException("too many blocked servers"); + Set unique = new HashSet<>(); + for (String server : blockedServers) { + if (server.isBlank() || server.length() > 100 || !unique.add(server.toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException("blocked server names must be unique and 1 to 100 characters"); + } + } + blockedServers = List.copyOf(blockedServers); + } + + public String revision() { + String canonical = sendVotesToAllServers + "\n" + String.join("\n", blockedServers.stream() + .map(value -> value.toLowerCase(Locale.ROOT)).sorted().toList()); + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(canonical.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + public List changesFrom(ProxyRoutingConfiguration current) { + List changes = new ArrayList<>(); + if (sendVotesToAllServers != current.sendVotesToAllServers) { + changes.add("sendVotesToAllServers: " + current.sendVotesToAllServers + " -> " + sendVotesToAllServers); + } + if (!blockedServers.equals(current.blockedServers)) { + changes.add("blockedServers: " + current.blockedServers + " -> " + blockedServers); + } + return List.copyOf(changes); + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java new file mode 100644 index 0000000000..835e6d40c8 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java @@ -0,0 +1,62 @@ +package com.bencodez.votingplugin.proxy.control; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Locale; +import java.util.Set; + +import com.bencodez.votingplugin.proxy.VotingPluginProxy; + +/** Validates, persists, reloads, and rolls back the bounded Control configuration domain. */ +public final class ProxyRoutingConfigurationService { + private final VotingPluginProxy proxy; + + public ProxyRoutingConfigurationService(VotingPluginProxy proxy) { + this.proxy = proxy; + } + + public ProxyRoutingConfiguration read() { + return new ProxyRoutingConfiguration(proxy.getConfig().getSendVotesToAllServers(), + proxy.getConfig().getBlockedServers()); + } + + public void validate(ProxyRoutingConfiguration proposal) { + Set available = new HashSet<>(); + proxy.getAllAvailableServers().forEach(server -> available.add(server.toLowerCase(Locale.ROOT))); + for (String server : proposal.blockedServers()) { + if (!available.contains(server.toLowerCase(Locale.ROOT))) { + throw new IllegalArgumentException("unknown backend: " + server); + } + } + } + + public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) throws IOException { + validate(proposal); + if (expectedRevision == null || !read().revision().equals(expectedRevision)) { + throw new StaleRevisionException(); + } + proxy.getConfig().persistControlProxyRouting(proposal.sendVotesToAllServers(), proposal.blockedServers()); + try { + proxy.reloadCore(false); + } catch (RuntimeException e) { + boolean rolledBack = false; + try { + proxy.getConfig().rollbackControlProxyRouting(); + proxy.reloadCore(false); + rolledBack = true; + } catch (Exception rollbackFailure) { + e.addSuppressed(rollbackFailure); + } + throw new ApplyFailureException(rolledBack, e); + } + } + + @SuppressWarnings("serial") + public static final class StaleRevisionException extends RuntimeException { } + @SuppressWarnings("serial") + public static final class ApplyFailureException extends RuntimeException { + private final boolean rolledBack; + private ApplyFailureException(boolean rolledBack, Throwable cause) { super(cause); this.rolledBack = rolledBack; } + public boolean rolledBack() { return rolledBack; } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index d2d53b587a..5c2d550601 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -1,6 +1,10 @@ package com.bencodez.votingplugin.proxy.velocity; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -14,11 +18,13 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import org.spongepowered.configurate.ConfigurationNode; +import org.spongepowered.configurate.yaml.YamlConfigurationLoader; /** * Configuration file handler for Velocity proxy. */ public class VelocityConfig extends VelocityYMLFile implements VotingPluginProxyConfig { + private final File configurationFile; /** * Constructs a new Velocity configuration. @@ -26,6 +32,45 @@ public class VelocityConfig extends VelocityYMLFile implements VotingPluginProxy */ public VelocityConfig(File file) { super(file); + configurationFile = file; + } + + @Override + public synchronized void persistControlProxyRouting(boolean sendVotesToAllServers, List blockedServers) + throws IOException { + Path target = configurationFile.toPath(); + Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-stage"); + Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); + YamlConfigurationLoader sourceLoader = YamlConfigurationLoader.builder().path(target).build(); + ConfigurationNode candidate = sourceLoader.load(); + candidate.node("SendVotesToAllServers").set(sendVotesToAllServers); + candidate.node("BlockedServers").setList(String.class, List.copyOf(blockedServers)); + try { + YamlConfigurationLoader.builder().path(stage).build().save(candidate); + Files.copy(target, backup, StandardCopyOption.REPLACE_EXISTING); + atomicReplace(stage, target); + } finally { + Files.deleteIfExists(stage); + } + } + + @Override + public synchronized void rollbackControlProxyRouting() throws IOException { + Path target = configurationFile.toPath(); + Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); + if (!Files.isRegularFile(backup)) throw new IOException("Control backup is unavailable"); + Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); + Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); + atomicReplace(stage, target); + reload(); + } + + private static void atomicReplace(Path source, Path target) throws IOException { + try { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (java.nio.file.AtomicMoveNotSupportedException e) { + throw new IOException("Atomic Control configuration activation is unsupported", e); + } } @Override diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index a517a5bb90..5e0dbf919a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -43,7 +43,7 @@ class ControlConnectorTest { @Test void registrationHeartbeatAndPresenceAreWireCompatibleAndReplacementSnapshotsAdvance() { connector.cycle(); assertEquals(Status.CONNECTED, connector.status()); - assertEquals(2, transport.requests.size()); + assertEquals(3, transport.requests.size()); Request registration = transport.requests.get(0); assertEquals("POST", registration.method()); assertEquals("/api/v1/nodes/register", registration.path()); @@ -59,8 +59,8 @@ class ControlConnectorTest { .get("backendId").getAsString()); connector.cycle(); - assertEquals("/api/v1/nodes/proxy-a/heartbeat", transport.requests.get(2).path()); - JsonObject secondSnapshot = JsonParser.parseString(transport.requests.get(3).body()).getAsJsonObject(); + assertEquals("/api/v1/nodes/proxy-a/heartbeat", transport.requests.get(3).path()); + JsonObject secondSnapshot = JsonParser.parseString(transport.requests.get(4).body()).getAsJsonObject(); assertEquals(1, secondSnapshot.get("sequence").getAsLong()); } @@ -89,8 +89,8 @@ class ControlConnectorTest { connector.cycle(); assertEquals(Status.UNAVAILABLE, connector.status()); connector.cycle(); - assertEquals("POST", transport.requests.get(3).method()); - assertEquals("/api/v1/nodes/register", transport.requests.get(3).path()); + assertEquals("POST", transport.requests.get(4).method()); + assertEquals("/api/v1/nodes/register", transport.requests.get(4).path()); } @Test void slowTransportDoesNotBlockCallerAndShutdownCancelsInFlightRequest() { @@ -104,6 +104,14 @@ class ControlConnectorTest { assertEquals(Status.STOPPED, connector.status()); } + @Test void synchronousTransportFailureClearsInFlightAndRetries() { + transport.synchronousFailure = new IllegalArgumentException("invalid header"); + connector.cycle(); + assertEquals(Status.UNAVAILABLE, connector.status()); + connector.cycle(); + assertEquals(Status.CONNECTED, connector.status()); + } + @Test void backoffIsBoundedExponentialAndJitteredWithoutSleeping() { assertEquals(1000, ControlConnector.backoffMillis(1, 0)); assertEquals(2000, ControlConnector.backoffMillis(2, 0)); @@ -138,22 +146,31 @@ private static final class FakeTransport implements Transport { private final List requests = new ArrayList<>(); private Response nextPrimary; private CompletableFuture stalled; + private RuntimeException synchronousFailure; @Override public CompletableFuture send(Request request) { + if (synchronousFailure != null) { + RuntimeException failure = synchronousFailure; + synchronousFailure = null; + throw failure; + } requests.add(request); if (stalled != null) { CompletableFuture result = stalled; stalled = null; return result; } + if (request.path().endsWith("/operations")) { + return CompletableFuture.completedFuture(new Response(204, "")); + } if (!request.path().endsWith("/presence")) { Response response = nextPrimary; nextPrimary = null; if (response != null) { return CompletableFuture.completedFuture(response); } - if ("POST".equals(request.method())) { + if ("/api/v1/nodes/register".equals(request.path())) { return CompletableFuture.completedFuture(new Response(201, "{\"identity\":{\"protocolVersion\":1},\"node\":{\"acceptedCapabilities\":[\"presence.snapshot\"]}}")); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 34edc43199..5585f0906b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -91,6 +91,10 @@ void failedUpdateRollsBackAndStartsPreviousRelease() throws Exception { assertEquals("previous-release", Files.readString(jar)); assertEquals("bad-new-release", Files.readString(settings.failedFile())); assertTrue(launcher.processes.get(0).destroyed); + Files.writeString(jar, "corrupted-after-rollback"); + manager.runOnce(); + assertEquals(HostedControlManager.Status.FAILED, manager.status()); + assertEquals(2, launcher.launchedContents.size()); manager.close(); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationTest.java new file mode 100644 index 0000000000..0a80cd3868 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationTest.java @@ -0,0 +1,24 @@ +package com.bencodez.votingplugin.proxy.control; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.util.List; +import org.junit.jupiter.api.Test; + +class ProxyRoutingConfigurationTest { + @Test void revisionIsStableAcrossBlockedServerCaseAndOrder() { + ProxyRoutingConfiguration first = new ProxyRoutingConfiguration(true, List.of("Lobby", "survival")); + ProxyRoutingConfiguration second = new ProxyRoutingConfiguration(true, List.of("SURVIVAL", "lobby")); + assertEquals(first.revision(), second.revision()); + } + + @Test void rejectsDuplicateBlankAndOversizedServerLists() { + assertThrows(IllegalArgumentException.class, + () -> new ProxyRoutingConfiguration(true, List.of("Lobby", "lobby"))); + assertThrows(IllegalArgumentException.class, + () -> new ProxyRoutingConfiguration(true, List.of(" "))); + assertThrows(IllegalArgumentException.class, + () -> new ProxyRoutingConfiguration(true, java.util.Collections.nCopies(257, "lobby"))); + } +} diff --git a/docs/control-connector.md b/docs/control-connector.md index 7b72d3f8f1..1f8b9d3b13 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -1,10 +1,11 @@ # VotingPlugin Control discovery connector -This development milestone adds an optional, read-only connector from the BungeeCord and Velocity VotingPlugin proxy JAR +This development milestone adds an optional connector from the BungeeCord and Velocity VotingPlugin proxy JAR to the separate [VotingPlugin Control](https://github.com/BenCodez/VotingPlugin-Control) application. It discovers each proxy and its eligible backend server names. Control includes the local WebUI. VotingPlugin may now explicitly opt in to provisioning and supervising that application as a separate child JVM; it is never loaded into the proxy classloader. The -connector does not expose backend addresses or configuration and does not connect directly from Bukkit backend nodes. +connector does not expose backend addresses, secrets, or unrestricted configuration and does not connect directly from +Bukkit backend nodes. ## Failure isolation @@ -121,7 +122,16 @@ presence protocol, the existing `BackendPlayerPresenceTracker` supplies availabi has no authoritative observation, the backend remains discoverable with `presenceKnown: false`; the connector does not invent an online state from a server address or arbitrary wall-clock timestamp. -The connector advertises and requires `presence.snapshot`. Authentication failures, unavailable required capabilities, +The connector requires `presence.snapshot` and advertises `config.proxy-routing.v1` when its typed adapter is available. +The latter permits reads and preview/apply of only `SendVotesToAllServers` and `BlockedServers`. Every preview validates +blocked names against that proxy's configured backends and returns a deterministic SHA-256 revision. Apply rejects stale +revisions, writes a local `.control-backup`, requires atomic activation of a staged YAML file, and soft-reloads the +proxy. A reload failure triggers immediate backup restoration and another reload attempt; Control receives the per-node +reload/rollback result. Task results are cached by operation ID so a leased retry cannot apply the same change twice. + +The WebUI never sends raw YAML. An admin must select capable online proxies, preview the typed change, and confirm the +single-use approval generated for that exact successful preview. Nodes claim work over their existing outbound connection, +so no inbound listener is added to VotingPlugin. Authentication failures, unavailable required capabilities, and protocol mismatches back off only the connector. Current redacted diagnostic status is one of `DISABLED`, `STARTING`, `CONNECTED`, `AUTHENTICATION_FAILED`, `INCOMPATIBLE`, `UNAVAILABLE`, or `STOPPED`. @@ -137,6 +147,6 @@ default. - Backend listed with unknown presence: this is expected when the selected existing VotingPlugin transport does not provide backend-presence observations. -Configuration reads/writes, topology persistence, cloud relay, diagnostics downloads, signed remote release manifests, -and remote support remain later milestones. Hosted downloads currently require the SHA-256 trust pin to be supplied in -local configuration; VotingPlugin never trusts a remotely fetched checksum by itself. +Unrestricted configuration, manual rollback, topology persistence, cloud relay, diagnostics downloads, signed remote +release manifests, and remote support remain later milestones. Hosted downloads currently require the SHA-256 trust pin +to be supplied in local configuration; VotingPlugin never trusts a remotely fetched checksum by itself. From ef57ba0495560a6169f4c13852fa08ffece29dbf Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 24 Aug 2026 20:38:25 -0600 Subject: [PATCH 05/73] Make Control configuration reloads fail safe --- .../votingplugin/proxy/VotingPluginProxy.java | 3245 +---------------- .../proxy/bungee/BungeeConfig.java | 41 +- .../proxy/bungee/VotingPluginBungee.java | 23 +- .../ProxyRoutingConfigurationService.java | 43 +- .../proxy/velocity/VelocityConfig.java | 8 +- .../proxy/velocity/VotingPluginVelocity.java | 13 + .../ProxyRoutingConfigurationServiceTest.java | 56 + .../tests/VotingPluginProxyTestImpl.java | 10 + docs/control-connector.md | 14 +- 9 files changed, 169 insertions(+), 3284 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 43c67b93b6..a6e507f63b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1,3244 +1 @@ -package com.bencodez.votingplugin.proxy; - -import java.io.ByteArrayOutputStream; -import java.io.DataInputStream; -import java.io.DataOutputStream; -import java.io.File; -import java.io.IOException; -import java.net.InetSocketAddress; -import java.net.Socket; -import java.net.URI; -import java.net.http.HttpClient; -import java.net.http.HttpRequest; -import java.net.http.HttpResponse; -import java.sql.SQLException; -import java.time.Duration; -import java.time.Instant; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.time.ZoneOffset; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -import org.eclipse.paho.client.mqttv3.MqttException; - -import com.bencodez.advancedcore.api.time.TimeType; -import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; -import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalMySQL; -import com.bencodez.advancedcore.bungeeapi.time.BungeeTimeChecker; -import com.bencodez.simpleapi.encryption.EncryptionHandler; -import com.bencodez.simpleapi.json.JsonParser; -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; -import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; -import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener; -import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; -import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; -import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; -import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; -import com.bencodez.simpleapi.servercomm.redis.RedisHandler; -import com.bencodez.simpleapi.servercomm.redis.RedisListener; -import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; -import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; -import com.bencodez.simpleapi.servercomm.sockets.SocketReceiver; -import com.bencodez.simpleapi.sql.Column; -import com.bencodez.simpleapi.sql.DataType; -import com.bencodez.simpleapi.sql.data.DataValue; -import com.bencodez.simpleapi.sql.data.DataValueBoolean; -import com.bencodez.simpleapi.sql.data.DataValueInt; -import com.bencodez.simpleapi.sql.data.DataValueString; -import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; -import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; -import com.bencodez.votingplugin.proxy.cache.IVoteCache; -import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; -import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; -import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; -import com.bencodez.votingplugin.proxy.control.ControlConnector; -import com.bencodez.votingplugin.proxy.control.HostedControlManager; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyMethod; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfiguration; -import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfigurationBungee; -import com.bencodez.votingplugin.proxy.presence.BackendPlayerPresenceTracker; -import com.bencodez.votingplugin.proxy.presence.PlayerPresence; -import com.bencodez.votingplugin.timequeue.VoteTimeQueue; -import com.bencodez.votingplugin.topvoter.TopVoter; -import com.bencodez.votingplugin.util.MinecraftUsernameValidator; -import com.bencodez.votingplugin.util.ServiceSiteValidator; -import com.bencodez.votingplugin.votelog.VoteLogMysqlTable; -import com.bencodez.votingplugin.votelog.VoteLogMysqlTable.VoteLogStatus; -import com.google.gson.JsonElement; -import com.google.gson.JsonObject; - -import redis.clients.jedis.DefaultJedisClientConfig; -import redis.clients.jedis.HostAndPort; -import redis.clients.jedis.Jedis; -import redis.clients.jedis.JedisPool; - -import lombok.Getter; -import lombok.Setter; - -public abstract class VotingPluginProxy { - private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); - private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; - private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; - private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); - - @Getter - @Setter - private int votePartyVotes = 0; - - @Getter - @Setter - private int currentVotePartyVotesRequired = 0; - - @Getter - @Setter - private ProxyMysqlUserTable proxyMySQL; - - private EncryptionHandler encryptionHandler; - - private HashMap clientHandles; - - private SocketHandler socketHandler; - - @Getter - @Setter - private boolean votifierEnabled = true; - - @Getter - private ConcurrentHashMap uuidPlayerNameCache = new ConcurrentHashMap<>(); - - @Getter - @Setter - private GlobalDataHandlerProxy globalDataHandler; - - @Getter - private RedisHandler redisHandler; - private JedisPool redisPublisherPool; - private volatile long redisPublisherRetryAfter; - private boolean timeVoteRetryScheduled; - private boolean timeVoteDeliveryRetryScheduled; - private boolean cachedVoteDeliveryRetryScheduled; - - private boolean enabled; - - @Getter - @Setter - private MultiProxyHandler multiProxyHandler; - - @Getter - private BungeeTimeChecker bungeeTimeChecker; - - @Getter - @Setter - private BungeeMethod method; - - @Getter - private MqttHandler mqttHandler; - - @Getter - private GlobalMessageProxyHandler globalMessageProxyHandler; - - @Getter - @Setter - private MySqlMessenger proxyMysqlMessenger; - - @Getter - private VoteCacheHandler voteCacheHandler; - - @Getter - private NonVotedPlayersCache nonVotedPlayersCache; - - @Getter - private final BackendPlayerPresenceTracker backendPlayerPresenceTracker = new BackendPlayerPresenceTracker(); - private final Map pendingPresenceHandoffs = new HashMap<>(); - private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); - private volatile ControlConnector controlConnector; - private volatile HostedControlManager hostedControlManager; - - public VotingPluginProxy() { - enabled = true; - - bungeeTimeChecker = new BungeeTimeChecker(getConfig().getTimeZone(), getConfig().getTimeHourOffSet(), - getConfig().getTimeWeekOffSet()) { - - @Override - public void debug(String text) { - debug2(text); - } - - @Override - public long getLastUpdated() { - return getVoteCacheLastUpdated(); - } - - @Override - public int getPrevDay() { - return getVoteCachePrevDay(); - } - - @Override - public String getPrevMonth() { - return getVoteCachePrevMonth(); - } - - @Override - public int getPrevWeek() { - return getVoteCachePrevWeek(); - } - - @Override - public void info(String text) { - log(text); - } - - @Override - public boolean isEnabled() { - return enabled; - } - - @Override - public boolean isIgnoreTime() { - return isVoteCacheIgnoreTime(); - } - - @Override - public void setIgnoreTime(boolean ignore) { - setVoteCacheVoteCacheIgnoreTime(ignore); - } - - @Override - public void setLastUpdated() { - setVoteCacheLastUpdated(); - } - - @Override - public void setPrevDay(int day) { - setVoteCachePrevDay(day); - } - - @Override - public void setPrevMonth(String text) { - setVoteCachePrevMonth(text); - } - - @Override - public void setPrevWeek(int week) { - setVoteCachePrevWeek(week); - } - - @Override - public void timeChanged(TimeType type, boolean fake, boolean pre, boolean post) { - if (getConfig().getVoteCacheTime() > 0) { - getVoteCacheHandler().checkVoteCacheTime(getConfig().getVoteCacheTime()); - } - if (!getConfig().getGlobalDataEnabled()) { - warn("Global data not enabled, ignoring time change event"); - return; - } - int delay = 1; - for (String s : getAllAvailableServers()) { - if (getGlobalDataHandler().getGlobalMysql().containsKey(s)) { - String lastOnlineStr = getGlobalDataHandler().getString(s, "LastOnline"); - long lastOnline = 0; - try { - lastOnline = Long.valueOf(lastOnlineStr); - } catch (NumberFormatException e) { - // ignore - } - - if (LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli() - lastOnline < 1000 - * 60 * 60 * 12) { - HashMap dataToSet = new HashMap<>(); - dataToSet.put("LastUpdated", new DataValueString( - "" + LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli())); - dataToSet.put("FinishedProcessing", new DataValueBoolean(false)); - dataToSet.put(type.toString(), new DataValueBoolean(true)); - getGlobalDataHandler().setData(s, dataToSet); - - globalMessageProxyHandler.sendMessage(s, delay, VotingPluginWire.bungeeTimeChange()); - delay++; - } else { - warn("Server " + s + " hasn't been online recently"); - } - } else { - warn("Server " + s + " global data handler disabled?"); - } - } - globalDataHandler.onTimeChange(type); - } - - @Override - public void warning(String text) { - warn(text); - } - }; - } - - public void onTimeChangedFailed(String srv, TimeType type) { - getGlobalDataHandler().setBoolean(srv, type.toString(), false); - getGlobalDataHandler().setBoolean(srv, "FinishedProcessing", true); - getGlobalDataHandler().setBoolean(srv, "Processing", false); - } - - public void onTimeChangedFinished(TimeType type) { - if (type.equals(TimeType.MONTH)) { - getProxyMySQL().copyColumnData(TopVoter.Monthly.getColumnName(), "LastMonthTotal"); - } - getProxyMySQL().wipeColumnData(TopVoter.of(type).getColumnName(), DataType.INTEGER); - - if (!getConfig().getGlobalDataEnabled()) { - return; - } - for (String s : getAllAvailableServers()) { - getGlobalDataHandler().setBoolean(s, "ForceUpdate", true); - getGlobalMessageProxyHandler().sendMessage(s, 1, VotingPluginWire.bungeeTimeChange()); - } - processQueue(); - } - - /** - * Load MySQL + global data handler. - */ - public void loadMysql(MysqlConfig mysqlConfig, MysqlConfig globalDataMysqlConfig) { - if (mysqlConfig.getHostName().isEmpty() || mysqlConfig.getDatabase().isEmpty()) { - logSevere("MySQL is not configured correctly. " + "Missing host/database. host=" + mysqlConfig.getHostName() - + " db=" + mysqlConfig.getDatabase()); - setProxyMySQL(null); - return; - } - - setProxyMySQL(new ProxyMysqlUserTable("VotingPlugin_Users", mysqlConfig, getConfig().getDebug()) { - - @Override - public void debug(SQLException e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable t) { - if (getConfig().getDebug()) { - t.printStackTrace(); - } - } - - @Override - public void debug(String str) { - debug2(str); - } - }); - - ArrayList servers = new ArrayList(getAllAvailableServers()); - - if (getConfig().getGlobalDataEnabled()) { - if (getConfig().getGlobalDataUseMainMySQL()) { - setGlobalDataHandler(new GlobalDataHandlerProxy( - new GlobalMySQL("VotingPlugin_GlobalData", getProxyMySQL().getMysql()) { - - @Override - public void debugEx(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debugLog(String text) { - debug(text); - } - - @Override - public void info(String text) { - logInfo(text); - } - - @Override - public void logSevere(String text) { - VotingPluginProxy.this.logSevere(text); - } - - @Override - public void warning(String text) { - warn(text); - } - }, servers) { - - @Override - public void onTimeChangedFailed(String srv, TimeType type) { - VotingPluginProxy.this.onTimeChangedFailed(srv, type); - } - - @Override - public void onTimeChangedFinished(TimeType type) { - VotingPluginProxy.this.onTimeChangedFinished(type); - } - }); - } else { - setGlobalDataHandler( - new GlobalDataHandlerProxy(new GlobalMySQL("VotingPlugin_GlobalData", globalDataMysqlConfig) { - - @Override - public void debugEx(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debugLog(String text) { - debug(text); - } - - @Override - public void info(String text) { - logInfo(text); - } - - @Override - public void logSevere(String text) { - VotingPluginProxy.this.logSevere(text); - } - - @Override - public void warning(String text) { - warn(text); - } - }, servers) { - - @Override - public void onTimeChangedFailed(String srv, TimeType type) { - VotingPluginProxy.this.onTimeChangedFailed(srv, type); - } - - @Override - public void onTimeChangedFinished(TimeType type) { - VotingPluginProxy.this.onTimeChangedFinished(type); - } - }); - } - - // update global schema columns (unchanged from original) - getGlobalDataHandler().getGlobalMysql().alterColumnType("IgnoreTime", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("MONTH", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("WEEK", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("DAY", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("FinishedProcessing", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("Processing", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("ForceUpdate", "VARCHAR(5)"); - getGlobalDataHandler().getGlobalMysql().alterColumnType("LastUpdated", "MEDIUMTEXT"); - } - - // column types (unchanged from original) - getProxyMySQL().alterColumnType("TopVoterIgnore", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("CheckWorld", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("Reminded", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("DisableBroadcast", "VARCHAR(5)"); - getProxyMySQL().alterColumnType("LastOnline", "VARCHAR(20)"); - getProxyMySQL().alterColumnType("PlayerName", "VARCHAR(30)"); - getProxyMySQL().alterColumnType("DailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("WeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("DayVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("BestDayVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("WeekVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("BestWeekVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("VotePartyVotes", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("MonthVoteStreak", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("Points", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestDailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("AllTimeTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestMonthlyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("MonthTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("HighestWeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastMonthTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastWeeklyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("LastDailyTotal", "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType("OfflineRewards", "MEDIUMTEXT"); - getProxyMySQL().alterColumnType("DayVoteStreakLastUpdate", "MEDIUMTEXT"); - - if (getConfig().getStoreMonthTotalsWithDate()) { - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now()), "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(1)), - "INT DEFAULT '0'"); - getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(2)), - "INT DEFAULT '0'"); - } - } - - public void addCurrentVotePartyVotes(int amount) { - votePartyVotes += amount; - setVoteCacheVotePartyCurrentVotes(votePartyVotes); - debug("Current vote party total: " + votePartyVotes); - } - - public void addNonVotedPlayer(String uuid, String playerName) { - nonVotedPlayersCache.addPlayer(uuid, playerName); - } - - public void addVoteParty() { - if (getConfig().getVotePartyEnabled()) { - addCurrentVotePartyVotes(1); - checkVoteParty(); - } - } - - public abstract void broadcast(String message); - - private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, - String text, boolean wasOnline) { - Set forwarded = new LinkedHashSet<>(); - for (String targetServer : targets) { - JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); - if (sendProxyBroadcastEnvelopeNow(targetServer, envelope)) { - forwarded.add(targetServer); - } - } - return forwarded; - } - - /** - * Sends a standalone proxy broadcast through the selected transport and reports - * whether that transport accepted the message. - * - * @param server target backend server - * @param envelope standalone broadcast envelope - * @return true only when the transport accepted the message - */ - protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope envelope) { - switch (method) { - case MQTT: - return sendMqttEnvelopeServer(server, envelope); - case MYSQL: - if (proxyMysqlMessenger == null) { - return false; - } - try { - proxyMysqlMessenger.sendToBackend(server, envelope); - return true; - } catch (SQLException e) { - debug(e.getMessage()); - return false; - } - case PLUGINMESSAGING: - return sendPluginMessageServerNow(server, envelope); - case REDIS: - return sendRedisEnvelopeServer(server, envelope, true); - case SOCKETS: - // Standalone broadcasts use the same initialized client as normal - // envelopes. This preserves the socket connection and its delivery - // acknowledgement instead of creating a second short-lived socket. - ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); - if (socketClient == null) { - return false; - } - try { - socketClient.sendEnvelope(envelope); - return true; - } catch (RuntimeException e) { - debug(e.getMessage()); - return false; - } - default: - return false; - } - } - - public synchronized void checkCachedVotes(String server) { - int delay = 1; - if (isServerValid(server)) { - if (isSomeoneOnlineServerForVoteRouting(server)) { - if (getVoteCacheHandler().hasVotes(server) && !getConfig().getBlockedServers().contains(server)) { - ArrayList c = getVoteCacheHandler().getVotes(server); - ArrayList removed = new ArrayList<>(); - if (!c.isEmpty()) { - int num = 1; - int numberOfVotes = c.size(); - for (OfflineBungeeVote cache : c) { - if (cache.isDeliveryStateDirty() && !persistServerVoteDelivery(server, cache)) { - continue; - } - if (cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server)) { - Set forwarded = sendProxyBroadcast(Collections.singleton(server), - cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), - cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (!persistServerVoteDelivery(server, cache)) { - continue; - } - } - } - - boolean toSend = true; - if (getConfig().getWaitForUserOnline()) { - if (!isPlayerOnlineForVoteRouting(cache.getPlayerName())) { - toSend = false; - } else if (isPlayerOnlineForVoteRouting(cache.getPlayerName()) - && !getCurrentPlayerServerForVoteRouting(cache.getPlayerName()).equals(server)) { - toSend = false; - } - } - if (toSend) { - boolean broadcastHere = cache.needsBroadcastOn(server); - if (!cache.isProxyBroadcastHandled() && broadcastHere - && getConfig().getProxyBroadcastEnabled()) { - boolean playerOnline = isPlayerOnlineForVoteRouting(cache.getPlayerName()); - String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(cache.getPlayerName()) - : null; - - Set targets = proxyBroadcastDecider.resolveTargets(playerOnline, - playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - globalMessageProxyHandler.sendMessage(server, delay, - VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), - cache.getService(), cache.getTime(), false, cache.isRealVote(), - cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), - broadcastHere, num, numberOfVotes)); - delay++; - num++; - removed.add(cache); - } else { - debug("Not sending vote because user isn't on server " + server + ": " - + cache.toString()); - } - } - getVoteCacheHandler().removeServerVotes(server, removed); - } else { - debug("No cached votes for server: " + server); - } - } else { - debug("No cached votes for server: " + server); - } - } - } else { - debug("Server not valid: " + server); - } - } - - public synchronized void checkOnlineVotes(String player, String uuid, String server) { - int delay = 1; - if (isPlayerOnlineForVoteRouting(player) && getVoteCacheHandler().hasOnlineVotes(uuid)) { - ArrayList c = getVoteCacheHandler().getOnlineVotes(uuid); - if (!c.isEmpty()) { - if (server == null) { - server = getCurrentPlayerServerForVoteRouting(player); - } - if (!getConfig().getBlockedServers().contains(server)) { - int num = 1; - int numberOfVotes = (int) c.stream().filter(vote -> !vote.isRewardDelivered()).count(); - boolean deliveredReward = false; - ArrayList retained = new ArrayList<>(); - for (OfflineBungeeVote cache : c) { - if (cache.isProxyBroadcastHandled()) { - Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); - pendingTargets.removeAll(cache.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, - cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), - cache.getText(), false)); - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - } - boolean broadcastHere = cache.needsBroadcastOn(server); - if (!cache.isProxyBroadcastHandled() && broadcastHere - && getConfig().getProxyBroadcastEnabled()) { - String playerServer = (server != null) ? server : getCurrentPlayerServerForVoteRouting(player); - - Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - if (!cache.isRewardDelivered()) { - globalMessageProxyHandler.sendMessage(server, delay, - VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), - cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), - getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes)); - // The normal envelope is also a valid broadcast delivery for the - // current target. Record it so a previously pending standalone - // retry cannot announce the same vote again later. - if (cache.isProxyBroadcastHandled() && broadcastHere) { - cache.getBroadcastForwardedServers().add(server); - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - } - cache.setRewardDelivered(true); - deliveredReward = true; - delay++; - num++; - } - - if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { - retained.add(cache); - } - } - getVoteCacheHandler().removeOnlineVotes(uuid); - for (OfflineBungeeVote pending : retained) { - getVoteCacheHandler().addOnlineVote(uuid, pending); - } - - // multiproxy: envelope-only - if (deliveredReward && getConfig().getMultiProxySupport() - && getConfig().getMultiProxyOneGlobalReward()) { - multiProxyHandler.sendClearVote(uuid, player); - } - } - } - } - } - - /** - * Retries voter-keyed standalone broadcasts when any player makes a target - * backend available as a plugin-message carrier. - * - * @param server backend server that gained a carrier - */ - protected synchronized void retryPendingOnlineBroadcasts(String server) { - List blockedServers = getConfig().getBlockedServers(); - if (server == null || (blockedServers != null && blockedServers.contains(server))) { - return; - } - for (String cachedUuid : getVoteCacheHandler().getOnlineVoteUUIDs()) { - for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { - if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { - continue; - } - if (!cache.isProxyBroadcastHandled() || !cache.needsBroadcastOn(server)) { - continue; - } - Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(), - cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); - } else { - persistOnlineVoteDelivery(cachedUuid, cache); - } - } - } - } - } - - protected synchronized void retryPendingTimeBroadcasts(String server) { - List blockedServers = getConfig().getBlockedServers(); - if (server == null || (blockedServers != null && blockedServers.contains(server))) { - return; - } - if (getVoteCacheHandler().getTimeChangeQueue() == null) { - return; - } - for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { - if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { - continue; - } - if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty() || !vote.getBroadcastTargets().contains(server) - || vote.getBroadcastForwardedServers().contains(server)) { - continue; - } - Set forwarded = sendProxyBroadcast(Collections.singleton(server), vote.getUuid(), vote.getName(), - vote.getService(), vote.getTime(), vote.getTotals(), false); - if (vote.getBroadcastForwardedServers().addAll(forwarded)) { - persistTimeVoteDelivery(vote); - } - } - } - - /** - * Periodically retries every pending voter-keyed standalone broadcast. This is - * required for broker transports whose recovery does not produce a player-login - * carrier event. - */ - public synchronized void retryPendingOnlineBroadcasts() { - for (String cachedUuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { - for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { - if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { - continue; - } - if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { - continue; - } - Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); - pendingTargets.removeAll(cache.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - Set forwarded = sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), - cache.getService(), cache.getTime(), cache.getText(), false); - if (cache.getBroadcastForwardedServers().addAll(forwarded)) { - cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); - if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { - getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); - } else { - persistOnlineVoteDelivery(cachedUuid, cache); - } - } - } - } - retryPendingTimeBroadcasts(); - } - - public synchronized void retryPendingTimeBroadcasts() { - if (getVoteCacheHandler().getTimeChangeQueue() == null) { - return; - } - for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { - if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { - continue; - } - if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty()) { - continue; - } - Set pendingTargets = new LinkedHashSet<>(vote.getBroadcastTargets()); - pendingTargets.removeAll(vote.getBroadcastForwardedServers()); - List blockedServers = getConfig().getBlockedServers(); - if (blockedServers != null) { - pendingTargets.removeAll(blockedServers); - } - Set forwarded = sendProxyBroadcast(pendingTargets, vote.getUuid(), vote.getName(), vote.getService(), - vote.getTime(), vote.getTotals(), false); - if (vote.getBroadcastForwardedServers().addAll(forwarded)) { - persistTimeVoteDelivery(vote); - } - } - } - - protected synchronized boolean persistTimeVoteDelivery(VoteTimeQueue vote) { - if (getVoteCacheHandler().updateTimeVote(vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleTimeVoteDeliveryRetry(); - return false; - } - - private void scheduleTimeVoteDeliveryRetry() { - if (timeVoteDeliveryRetryScheduled || getScheduler() == null) { - return; - } - timeVoteDeliveryRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - timeVoteDeliveryRetryScheduled = false; - } - retryPendingTimeBroadcasts(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - timeVoteDeliveryRetryScheduled = false; - debug("Unable to schedule timed broadcast state retry: " + e.getMessage()); - } - } - - protected synchronized boolean persistServerVoteDelivery(String server, OfflineBungeeVote vote) { - if (getVoteCacheHandler().updateServerVote(server, vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleCachedVoteDeliveryRetry(); - return false; - } - - protected synchronized boolean persistOnlineVoteDelivery(String uuid, OfflineBungeeVote vote) { - if (getVoteCacheHandler().updateOnlineVote(uuid, vote)) { - vote.setDeliveryStateDirty(false); - return true; - } - vote.setDeliveryStateDirty(true); - scheduleCachedVoteDeliveryRetry(); - return false; - } - - private void scheduleCachedVoteDeliveryRetry() { - if (cachedVoteDeliveryRetryScheduled || getScheduler() == null) { - return; - } - cachedVoteDeliveryRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - cachedVoteDeliveryRetryScheduled = false; - } - retryCachedVoteDeliveryPersistence(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - cachedVoteDeliveryRetryScheduled = false; - debug("Unable to schedule cached broadcast state retry: " + e.getMessage()); - } - } - - private synchronized void retryCachedVoteDeliveryPersistence() { - for (String server : getVoteCacheHandler().getCachedVotesServers()) { - for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getVotes(server))) { - if (vote.isDeliveryStateDirty()) { - persistServerVoteDelivery(server, vote); - } - } - } - for (String uuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { - for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(uuid))) { - if (vote.isDeliveryStateDirty()) { - persistOnlineVoteDelivery(uuid, vote); - } - } - } - } - - public void checkVoteParty() { - if (getConfig().getVotePartyEnabled()) { - if (votePartyVotes >= currentVotePartyVotesRequired) { - debug("Vote party reached"); - addCurrentVotePartyVotes(-currentVotePartyVotesRequired); - - currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); - setVoteCacheVotePartyIncreaseVotesRequired( - getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired()); - - if (!getConfig().getVotePartyBroadcast().isEmpty()) { - broadcast(getConfig().getVotePartyBroadcast()); - } - - for (String command : getConfig().getVotePartyBungeeCommands()) { - runConsoleCommand(command); - } - - if (getConfig().getVotePartySendToAllServers()) { - for (String server : getAllAvailableServers()) { - sendVoteParty(server); - } - } else { - for (String server : getConfig().getVotePartyServersToSend()) { - sendVoteParty(server); - } - } - } - saveVoteCacheFile(); - } - } - - public abstract void debug(String str); - - private void debug2(String message) { - debug(message); - } - - /** - * HTTP client used for Mojang API requests. - */ - private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); - - /** - * Fetches a player's UUID from the Mojang API. - * - * @param playerName player name - * @return player UUID, or {@code null} if not found - * @throws IOException if the request fails - * @throws InterruptedException if interrupted while waiting for the response - */ - public UUID fetchUUID(String playerName) throws IOException, InterruptedException { - if (playerName == null || playerName.equalsIgnoreCase("null")) { - return null; - } - - HttpRequest request = HttpRequest.newBuilder() - .uri(URI.create("https://api.mojang.com/users/profiles/minecraft/" + playerName)).GET() - .timeout(Duration.ofSeconds(5)).build(); - - HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); - - if (response.statusCode() == 400 || response.statusCode() == 404) { - log("There is no player with the name \"" + playerName + "\"!"); - return null; - } - - if (response.statusCode() < 200 || response.statusCode() >= 300) { - throw new IOException("Failed to fetch UUID for " + playerName + ", HTTP " + response.statusCode()); - } - - JsonElement element = JsonParser.parseString(response.body()); - if (element == null || !element.isJsonObject()) { - return null; - } - - JsonObject object = element.getAsJsonObject(); - if (!object.has("id") || object.get("id").isJsonNull()) { - return null; - } - - String uuidAsString = object.get("id").getAsString(); - return parseUUIDFromString(uuidAsString); - } - - public abstract Set getAllAvailableServers(); - - public abstract VotingPluginProxyConfig getConfig(); - - public abstract String getCurrentPlayerServer(String player); - - /** - * Resolves a player's server for vote routing. A dedicated voting proxy has no - * local players, so it uses the backend presence tracker instead. - */ - protected String getCurrentPlayerServerForVoteRouting(String player) { - if (isDedicatedVotingProxyEnabled()) { - return backendPlayerPresenceTracker.getPlayer(player).map(presence -> presence.getServer()).orElse(null); - } - return getCurrentPlayerServer(player); - } - - /** - * Dedicated routing is intentionally unavailable on plugin messaging: that - * transport is attached to a player-facing proxy and does not carry backend - * presence snapshots. - */ - protected boolean isDedicatedVotingProxyEnabled() { - return getConfig().getDedicatedVotingProxy() && method != null && method.supportsBackendPresence(); - } - - public abstract File getDataFolderPlugin(); - - public String getMonthTotalsWithDatePath() { - LocalDateTime cTime = getBungeeTimeChecker().getTime(); - return getMonthTotalsWithDatePath(cTime); - } - - public String getMonthTotalsWithDatePath(LocalDateTime cTime) { - return "MonthTotal-" + cTime.getMonth().toString() + "-" + cTime.getYear(); - } - - public abstract String getProperName(String uuid, String playerName); - - public abstract String getUUID(String playerName); - - private int getValue(ArrayList cols, String column, int toAdd) { - for (Column d : cols) { - if (d.getName().equalsIgnoreCase(column)) { - DataValue value = d.getValue(); - int num = 0; - if (value == null) { - return toAdd; - } - if (value.isInt()) { - num = value.getInt(); - } else if (value.isString()) { - try { - num = Integer.parseInt(value.getString()); - } catch (Exception e) { - // ignore - } - } - return num + toAdd; - } - } - return toAdd; - } - - private VoteTotalsSnapshot getProjectedRolloverTotals(ArrayList data, String player) { - List timeChanges = getGlobalDataHandler().getTimeChanges(); - boolean resetMonth = timeChanges.contains(TimeType.MONTH); - boolean resetWeek = timeChanges.contains(TimeType.WEEK); - boolean resetDay = timeChanges.contains(TimeType.DAY); - int acceptedQueuedVotes = 0; - int acceptedGlobalQueuedVotes = 0; - for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { - if (!queued.isProcessed()) { - acceptedGlobalQueuedVotes++; - } - if (!queued.isProcessed() && queued.getName() != null && queued.getName().equalsIgnoreCase(player)) { - acceptedQueuedVotes++; - } - } - int voteIncrement = acceptedQueuedVotes + 1; - - int allTimeTotal = getValue(data, "AllTimeTotal", voteIncrement); - int monthTotal = resetMonth ? voteIncrement : getValue(data, "MonthTotal", voteIncrement); - int weeklyTotal = resetWeek ? voteIncrement : getValue(data, "WeeklyTotal", voteIncrement); - int dailyTotal = resetDay ? voteIncrement : getValue(data, "DailyTotal", voteIncrement); - int points = getValue(data, "Points", voteIncrement * getConfig().getPointsOnVote()); - - int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); - if (maxVotes > 0) { - int days = getBungeeTimeChecker().getTime().getDayOfMonth(); - if (monthTotal > days * maxVotes) { - monthTotal = days * maxVotes; - } - } - if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { - points = getConfig().getLimitVotePoints(); - } - - int dateMonthTotal = -1; - if (getConfig().getStoreMonthTotalsWithDate()) { - if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { - dateMonthTotal = resetMonth ? voteIncrement - : getValue(data, getMonthTotalsWithDatePath(), voteIncrement); - } else { - dateMonthTotal = monthTotal; - } - } - - int[] projectedVoteParty = getProjectedVotePartyState(acceptedGlobalQueuedVotes + 1); - return new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, - projectedVoteParty[0], projectedVoteParty[1], dateMonthTotal); - } - - protected boolean canForwardStandaloneBroadcast(boolean managesTotals) { - return managesTotals; - } - - protected int[] getProjectedVotePartyState(int acceptedVotes) { - int current = votePartyVotes; - int required = currentVotePartyVotesRequired; - if (!getConfig().getVotePartyEnabled()) { - return new int[] { current, required }; - } - - int increase = getConfig().getVotePartyIncreaseVotesRequired(); - for (int i = 0; i < acceptedVotes; i++) { - current++; - if (current >= required) { - current -= required; - required += increase; - } - } - return new int[] { current, required }; - } - - public abstract String getPluginVersion(); - - public abstract int getVoteCacheCurrentVotePartyVotes(); - - public abstract long getVoteCacheLastUpdated(); - - public abstract int getVoteCachePrevDay(); - - public abstract String getVoteCachePrevMonth(); - - public abstract int getVoteCachePrevWeek(); - - public abstract int getVoteCacheVotePartyIncreaseVotesRequired(); - - public abstract boolean isPlayerOnline(String playerName); - - /** - * Checks online state for vote routing, using backend presence only when this - * proxy is explicitly configured as the dedicated voting proxy. - */ - protected boolean isPlayerOnlineForVoteRouting(String playerName) { - return isDedicatedVotingProxyEnabled() ? backendPlayerPresenceTracker.getPlayer(playerName).isPresent() - : isPlayerOnline(playerName); - } - - public abstract boolean isServerValid(String server); - - public abstract boolean isSomeoneOnlineServer(String server); - - protected boolean isSomeoneOnlineServerForVoteRouting(String server) { - if (!isDedicatedVotingProxyEnabled()) { - return isSomeoneOnlineServer(server); - } - com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus status = backendPlayerPresenceTracker - .getBackendStatus(server); - return status != null && status.isAvailable() && status.getPlayerCount() > 0; - } - - public abstract boolean isVoteCacheIgnoreTime(); - - public abstract MysqlConfig getVoteCacheMySQLConfig(); - - public abstract MysqlConfig getNonVotedCacheMySQLConfig(); - - public abstract MysqlConfig getVoteLoggingMySQLConfig(); - - /** - * Shutdown MySQL-related resources safely. - */ - public void shutdownMySql() { - if (getProxyMysqlMessenger() != null) { - getProxyMysqlMessenger().shutdown(); - setProxyMysqlMessenger(null); - } - - if (getProxyMySQL() != null) { - getProxyMySQL().shutdown(); - setProxyMySQL(null); - } - } - - public void load(IVoteCache jsonStorage, INonVotedPlayersStorage nonVotedCacheJson) { - method = BungeeMethod.getByName(getConfig().getBungeeMethod()); - if (getMethod() == null) { - method = BungeeMethod.PLUGINMESSAGING; - } - warnUnsupportedDedicatedVotingProxyMode(); - uuidPlayerNameCache = getProxyMySQL().getRowsUUIDNameQuery(); - - bungeeTimeChecker.setTimeChangeFailSafeBypass(getConfig().getTimeChangeFailSafeBypass()); - bungeeTimeChecker.loadTimer(); - - voteCacheHandler = new VoteCacheHandler(getVoteCacheMySQLConfig(), getConfig().getVoteCacheUseMySQL(), - getConfig().getVoteCacheUseMainMySQL(), getProxyMySQL().getMysql(), getConfig().getDebug(), - jsonStorage) { - - @Override - public void logInfo1(String msg) { - logInfo(msg); - } - - @Override - public void logSevere1(String msg) { - logSevere(msg); - } - - @Override - public void debug1(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debug1(String msg) { - if (getConfig().getDebug()) { - debug(msg); - } - } - - @Override - public void debug1(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - }; - voteCacheHandler.load(); - - nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), - getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(), - getProxyMySQL().getMysql(), nonVotedCacheJson, getConfig().getDebug()) { - - @Override - public boolean userExists(String uuid) { - return getProxyMySQL().containsKeyQuery(uuid); - } - - @Override - public void logInfo1(String msg) { - logInfo(msg); - } - - @Override - public void logSevere1(String msg) { - logSevere(msg); - } - - @Override - public void debug1(Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public void debug1(String msg) { - if (getConfig().getDebug()) { - debug(msg); - } - } - - @Override - public Set getAllUUIDs() { - return getProxyMySQL().getUuids(); - } - }; - - if (method.equals(BungeeMethod.MYSQL)) { - try { - proxyMysqlMessenger = new MySqlMessenger("VotingPlugin", - getProxyMySQL().getMysql().getConnectionManager().getDataSource(), MySqlMessenger.Mode.PROXY, - null, // no serverId in PROXY mode - msg -> { - if (getConfig().getDebug()) { - debug("Got from " + msg.source + ": " + msg.envelope.getSubChannel() + " " - + msg.envelope.getFields()); - } - globalMessageProxyHandler.onMessage(msg.envelope); - }); - } catch (SQLException e) { - e.printStackTrace(); - } - } else if (method.equals(BungeeMethod.PLUGINMESSAGING)) { - if (getConfig().getPluginMessageEncryption()) { - encryptionHandler = new EncryptionHandler("VotingPlugin", - new File(getDataFolderPlugin(), "secretkey.key")); - } - } else if (method.equals(BungeeMethod.SOCKETS)) { - encryptionHandler = new EncryptionHandler("VotingPlugin", new File(getDataFolderPlugin(), "secretkey.key")); - - socketHandler = new SocketHandler(getPluginVersion(), getConfig().getBungeeHost(), - getConfig().getBungeePort(), encryptionHandler, getConfig().getDebug()) { - - @Override - public void log(String str) { - logInfo(str); - } - }; - - socketHandler.add(new SocketReceiver() { - @Override - public void onReceiveEnvelope(JsonEnvelope envelope) { - globalMessageProxyHandler.onMessage(envelope); - } - }); - - clientHandles = new HashMap<>(); - List l = getConfig().getBlockedServers(); - for (String s : getConfig().getSpigotServers()) { - if (!l.contains(s)) { - Map d = getConfig().getSpigotServerConfiguration(s); - String host = ""; - if (d.containsKey("Host")) { - host = (String) d.get("Host"); - } - int port = 1298; - if (d.containsKey("Port")) { - port = (int) d.get("Port"); - } - clientHandles.put(s, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); - } - } - } else if (method.equals(BungeeMethod.REDIS)) { - redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), - getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex()) { - - @Override - public void debug(String message) { - debug2(message); - } - }; - redisPublisherPool = new JedisPool(new HostAndPort(getConfig().getRedisHost(), getConfig().getRedisPort()), - buildRedisClientConfig()); - - runAsync(() -> { - RedisListener listener = redisHandler.createEnvelopeListener( - getConfig().getRedisPrefix() + "VotingPlugin", - (ch, env) -> globalMessageProxyHandler.onMessage(env)); - redisHandler.loadListener(listener); - }); - - } else if (method.equals(BungeeMethod.MQTT)) { - try { - mqttHandler = new MqttHandler(new MqttServerComm(getConfig().getMqttClientID(), - getConfig().getMqttBrokerURL(), getConfig().getMqttUsername(), getConfig().getMqttPassword()), - 2); - - mqttHandler.subscribeEnvelopes(getConfig().getMqttPrefix() + "votingplugin/servers/proxy", - (topic, env) -> globalMessageProxyHandler.onMessage(env)); - - } catch (MqttException e) { - e.printStackTrace(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - currentVotePartyVotesRequired = getConfig().getVotePartyVotesRequired() - + getVoteCacheVotePartyIncreaseVotesRequired(); - votePartyVotes = getVoteCacheCurrentVotePartyVotes(); - - globalMessageProxyHandler = new GlobalMessageProxyHandler() { - @Override - public void sendMessage(String server, int delay, JsonEnvelope envelope) { - switch (method) { - case MQTT: - sendMqttEnvelopeServer(server, envelope); - break; - case MYSQL: - try { - proxyMysqlMessenger.sendToBackend(server, envelope); - } catch (SQLException e) { - e.printStackTrace(); - } - break; - case PLUGINMESSAGING: - sendPluginMessageServer(server, delay, envelope); - break; - case REDIS: - sendRedisEnvelopeServer(server, envelope); - break; - case SOCKETS: - ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); - if (socketClient != null) { - socketClient.sendEnvelope(envelope); - } - break; - default: - break; - } - } - }; - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGIN) { - @Override - public void onReceive(JsonEnvelope message) { - handleLoginMessage(message); - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGOUT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); - if (!isPresenceServerValid(event.server, VotingPluginWire.SUB_LOGOUT) - || !isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, - event.presenceTimestamp, - VotingPluginWire.SUB_LOGOUT)) { - return; - } - if (!backendPlayerPresenceTracker.playerOffline(event.uuid, event.server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, - System.currentTimeMillis())) { - debug("Ignored invalid or stale logout envelope: " + message.getFields()); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STARTED) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STARTED) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_STARTED)) { - if (backendPlayerPresenceTracker.backendStarted(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis())) { - discardPendingPresenceHandoffs(server); - pendingBackendRecoverySnapshots.add(presenceServerKey(server)); - requestBackendPresenceSnapshot(server); - } - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STOPPED) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STOPPED) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_STOPPED)) { - if (backendPlayerPresenceTracker.backendStopped(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis())) { - discardPendingPresenceHandoffs(server); - pendingBackendRecoverySnapshots.remove(presenceServerKey(server)); - } - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_HEARTBEAT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); - long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); - long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); - if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_HEARTBEAT) - && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, - VotingPluginWire.SUB_BACKEND_HEARTBEAT)) { - backendPlayerPresenceTracker.heartbeat(server, backendIncarnationId, backendStartedAt, - presenceTimestamp, System.currentTimeMillis()); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_PRESENCE_SNAPSHOT) { - @Override - public void onReceive(JsonEnvelope message) { - if (!method.supportsBackendPresence()) { - return; - } - String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); - if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT)) { - return; - } - VotingPluginWire.PresenceSnapshot snapshot = VotingPluginWire.readPresenceSnapshot(message); - long now = System.currentTimeMillis(); - boolean accepted = snapshot.valid - && isPresenceGenerationValid(snapshot.backendIncarnationId, snapshot.backendStartedAt, - snapshot.presenceTimestamp, - VotingPluginWire.SUB_PRESENCE_SNAPSHOT) - && backendPlayerPresenceTracker.applySnapshotChunk(snapshot.server, - snapshot.requestId, snapshot.chunkIndex, snapshot.chunkCount, snapshot.players, - snapshot.backendIncarnationId, snapshot.backendStartedAt, - snapshot.presenceTimestamp, now); - if (!accepted) { - debug("Ignored invalid or unexpected presence snapshot from " + snapshot.server); - if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { - discardPendingPresenceHandoffs(snapshot.requestId); - } - } else if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { - pendingBackendRecoverySnapshots.remove(presenceServerKey(snapshot.server)); - Set handoffPlayers = completePendingPresenceHandoffs(snapshot.requestId, snapshot.server, - snapshot.backendIncarnationId, snapshot.backendStartedAt, now); - processDedicatedSnapshotLogins(snapshot.server, handoffPlayers); - } - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener("statusokay") { - @Override - public void onReceive(JsonEnvelope message) { - String server = message.getFields().getOrDefault("server", ""); - log("Status okay for " + server); - } - }); - - globalMessageProxyHandler.addListener(new GlobalMessageListener("voteupdate") { - @Override - public void onReceive(JsonEnvelope message) { - int delay = 1; - for (String send : getAllAvailableServers()) { - globalMessageProxyHandler.sendMessage(send, delay, message); - delay++; - } - } - }); - - proxyBroadcastDecider = new ProxyBroadcastDecider(() -> getConfig(), () -> getAllAvailableServers(), - s -> isServerValid(s), - s -> getConfig().getBlockedServers() != null && getConfig().getBlockedServers().contains(s)); - - loadMultiProxySupport(); - loadVoteLoggingMySQL(); - if (method.supportsBackendPresence()) { - scheduleBackendPresenceStartupResync(); - loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, - PRESENCE_MAINTENANCE_INTERVAL_SECONDS); - } - startControlServices(); - - debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); - } - - private synchronized void startControlServices() { - stopControlServices(true); - if (getConfig().getControlHostedEnabled()) { - try { - hostedControlManager = HostedControlManager.create(this); - if (hostedControlManager != null) { - hostedControlManager.start(); - } - } catch (IllegalArgumentException e) { - hostedControlManager = null; - logSevere("[Control Host] configuration is invalid; VotingPlugin remains unaffected"); - } - } - if (!getConfig().getControlEnabled()) { - return; - } - try { - controlConnector = ControlConnector.create(this); - if (controlConnector != null) { - controlConnector.start(); - } - } catch (IOException | IllegalArgumentException e) { - controlConnector = null; - logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); - } - } - - private synchronized void stopControlServices(boolean waitForHosted) { - ControlConnector connector = controlConnector; - controlConnector = null; - if (connector != null) { - connector.close(); - } - HostedControlManager manager = hostedControlManager; - hostedControlManager = null; - if (manager != null) { - if (waitForHosted) { - manager.closeAndWait(); - } else { - manager.close(); - } - } - } - - public String getControlConnectorStatus() { - ControlConnector connector = controlConnector; - return connector == null ? "DISABLED" : connector.status().name(); - } - - public String getHostedControlStatus() { - HostedControlManager manager = hostedControlManager; - return manager == null ? "DISABLED" : manager.status().name(); - } - - /** - * Handles both the original login notification and extended presence logins. - * Kept protected so transport-policy behavior can be regression tested without - * initializing a live proxy transport. - * - * @param message login envelope - */ - protected void handleLoginMessage(JsonEnvelope message) { - VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); - String player = event.player; - String uuid = event.uuid; - String server = event.server; - - if (player.isEmpty() || uuid.isEmpty()) { - logSevere("Invalid login envelope received: " + message.getFields()); - return; - } - boolean legacy = event.connectionId == null && event.backendIncarnationId == null - && event.backendStartedAt == 0L && event.presenceTimestamp == 0L; - boolean accepted = false; - String deliveryServer = server; - if (legacy) { - if (method == BungeeMethod.PLUGINMESSAGING) { - String proxyServer = getCurrentPlayerServer(player); - accepted = isLegacyLoginDestinationAuthoritative(player, uuid, proxyServer); - if (accepted) { - deliveryServer = proxyServer; - } - } else if (method != null && method.supportsBackendPresence() - && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN)) { - accepted = isLegacyLoginDestinationAuthoritative(player, uuid, server); - } - } else if (method != null && method.supportsBackendPresence() && event.connectionId != null - && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN) - && isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, - event.presenceTimestamp, VotingPluginWire.SUB_LOGIN)) { - BackendPlayerPresenceTracker.PlayerOnlineResult result = backendPlayerPresenceTracker.playerOnlineResult( - player, uuid, server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, - System.currentTimeMillis()); - accepted = result.isAccepted(); - if (result.isConflictingPresence()) { - requestBackendPresenceSnapshot(server, - new PendingPresenceHandoff(player, uuid, server, event.connectionId, - event.backendIncarnationId, event.backendStartedAt, - result.getConflictSequence(), System.currentTimeMillis())); - } - } - - debug("Login: " + player + "/" + uuid + " " + server); - if (accepted) { - discardPendingPresenceHandoff(uuid); - login(player, uuid, deliveryServer); - } else { - debug("Ignored invalid or stale login envelope: " + message.getFields()); - } - } - - /** - * Validates a legacy login against an authority independent of the envelope. - * Player-facing proxies use their native live route and UUID. A dedicated - * voting proxy has no native player session, so it requires an exact modern - * presence match for the claimed destination. - */ - private boolean isLegacyLoginDestinationAuthoritative(String player, String uuid, String server) { - if (server == null || server.isBlank()) { - return false; - } - - UUID claimedUuid; - try { - claimedUuid = UUID.fromString(uuid.trim()); - } catch (RuntimeException e) { - return false; - } - - if (isDedicatedVotingProxyEnabled()) { - PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(player).orElse(null); - return presence != null && presence.getServer().equalsIgnoreCase(server) - && (!getConfig().getOnlineMode() || presence.getUuid().equals(claimedUuid)); - } - - if (!isPlayerOnline(player)) { - return false; - } - String proxyServer = getCurrentPlayerServer(player); - if (proxyServer == null || !proxyServer.equalsIgnoreCase(server)) { - return false; - } - if (!getConfig().getOnlineMode()) { - return true; - } - - String authoritativeUuid = getUUID(player); - if (authoritativeUuid == null || authoritativeUuid.isBlank()) { - return false; - } - try { - return claimedUuid.equals(UUID.fromString(authoritativeUuid.trim())); - } catch (IllegalArgumentException e) { - return false; - } - } - - private VoteLogMysqlTable voteLogMysqlTable; - - @Getter - private ProxyBroadcastDecider proxyBroadcastDecider; - - public void loadVoteLoggingMySQL() { - if (getConfig().getVoteLoggingEnabled()) { - if (getConfig().getVoteLoggingUseMainMySQL()) { - voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getProxyMySQL().getMysql(), - getVoteLoggingMySQLConfig(), getConfig().getDebug()) { - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public String getServerName() { - return "Proxy"; - } - }; - } else { - voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getVoteLoggingMySQLConfig(), - getConfig().getDebug()) { - - @Override - public void logSevere(String string) { - VotingPluginProxy.this.logSevere(string); - } - - @Override - public void logInfo(String string) { - VotingPluginProxy.this.logInfo(string); - } - - @Override - public void debug(Throwable e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - - @Override - public String getServerName() { - return "Proxy"; - } - }; - } - - if (getConfig().getVoteLoggingPurgeDays() > 0) { - loadTaskTimer(() -> voteLogMysqlTable.purgeOlderThanDays(getConfig().getVoteLoggingPurgeDays(), 100), - 60, 60 * 60); - } - - debug("Vote logging MySQL enabled"); - } else { - debug("Vote logging MySQL disabled"); - } - } - - public abstract void loadTaskTimer(Runnable runnable, long delaySeconds, long repeatSeconds); - - public void loadMultiProxySupport() { - if (multiProxyHandler != null) { - multiProxyHandler.close(); - } - multiProxyHandler = new MultiProxyHandler() { - - @Override - public void addNonVotedPlayerCache(String uuid, String player) { - addNonVotedPlayer(uuid, player); - } - - @Override - public void clearVote(String uuid) { - getVoteCacheHandler().clearOnlineVoteRewards(uuid); - } - - @Override - public boolean getDebug() { - return getConfig().getDebug(); - } - - @Override - public EncryptionHandler getEncryptionHandler() { - return encryptionHandler; - } - - @Override - public MultiProxyMethod getMultiProxyMethod() { - return MultiProxyMethod.getByName(getConfig().getMultiProxyMethod()); - } - - @Override - public String getMultiProxyPassword() { - return getConfig().getMultiProxyRedisPassword(); - } - - @Override - public String getMultiProxyRedisHost() { - return getConfig().getMultiProxyRedisHost(); - } - - @Override - public int getMultiProxyRedisPort() { - return getConfig().getMultiProxyRedisPort(); - } - - @Override - public int getMultiProxyRedisDbIndex() { - return getConfig().getMultiProxyRedisDbIndex(); - } - - @Override - public boolean getMultiProxyRedisUseExistingConnection() { - return getConfig().getMultiProxyRedisUseExistingConnection(); - } - - @Override - public String getMultiProxyServerName() { - return getConfig().getProxyServerName(); - } - - @Override - public Collection getMultiProxyServers() { - return getConfig().getMultiProxyServers(); - } - - @Override - public MultiProxyServerSocketConfiguration getMultiProxyServersConfiguration(String s) { - return new MultiProxyServerSocketConfigurationBungee(s, - getConfig().getMultiProxyServersConfiguration(s)); - } - - @Override - public String getMultiProxySocketHostHost() { - return getConfig().getMultiProxySocketHostHost(); - } - - @Override - public int getMultiProxySocketHostPort() { - return getConfig().getMultiProxySocketHostPort(); - } - - @Override - public boolean getMultiProxySupportEnabled() { - return getConfig().getMultiProxySupport(); - } - - @Override - public String getMultiProxyUsername() { - return getConfig().getMultiProxyRedisUsername(); - } - - @Override - public File getPluginDataFolder() { - return getDataFolderPlugin(); - } - - @Override - public boolean getPrimaryServer() { - return getConfig().getPrimaryServer(); - } - - @Override - public List getProxyServers() { - return getConfig().getProxyServers(); - } - - @Override - public RedisHandler getRedisHandler() { - return redisHandler; - } - - @Override - public String getVersion() { - return getPluginVersion(); - } - - @Override - public void logInfo(String msg) { - log(msg); - } - - @Override - public void runAsnc(Runnable runnable) { - runAsync(runnable); - } - - @Override - public void setEncryptionHandler(EncryptionHandler encryptionHandler1) { - encryptionHandler = encryptionHandler1; - } - - @Override - public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid) { - vote(player, service, realVote, timeQueue, queueTime, text, uuid); - } - }; - multiProxyHandler.loadMultiProxySupport(); - } - - public abstract void log(String message); - - /** - * Requests a complete player-presence snapshot from one backend server. - * - * @param server configured backend server name - * @return new or already-active request identifier, or null when the server is - * invalid or is inside the snapshot-request cooldown - */ - public UUID requestBackendPresenceSnapshot(String server) { - return requestBackendPresenceSnapshot(server, null); - } - - private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff) { - return requestBackendPresenceSnapshot(server, handoff, System.currentTimeMillis(), false); - } - - private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff, long now, - boolean handoffAlreadyQueued) { - if (method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null - || !isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT_REQUEST)) { - return null; - } - long backendStartedAt = backendPlayerPresenceTracker.getBackendStartedAt(server); - UUID backendIncarnationId = backendPlayerPresenceTracker.getBackendIncarnationId(server); - if (backendStartedAt <= 0L || backendIncarnationId == null) { - return null; - } - if (handoff != null && (!server.equalsIgnoreCase(handoff.server) - || !backendIncarnationId.equals(handoff.backendIncarnationId) - || backendStartedAt != handoff.backendStartedAt)) { - return null; - } - if (handoff != null && (handoffAlreadyQueued ? !isPendingPresenceHandoff(handoff, now) - : !queuePendingPresenceHandoff(handoff, now))) { - return null; - } - UUID requestId = handoff == null - ? backendPlayerPresenceTracker.beginSnapshot(server, UUID.randomUUID(), backendIncarnationId, - backendStartedAt, now) - : backendPlayerPresenceTracker.beginSnapshotForDestinationClaim(server, UUID.randomUUID(), - backendIncarnationId, backendStartedAt, handoff.playerUuid, handoff.conflictSequence, now); - boolean created = requestId != null; - if (!created) { - requestId = handoff == null ? backendPlayerPresenceTracker.getPendingSnapshotRequestId(server, now) - : backendPlayerPresenceTracker.getPendingSnapshotRequestIdForDestinationClaim(server, - handoff.playerUuid, handoff.conflictSequence, now); - } - if (requestId == null) { - if (handoff != null && !backendPlayerPresenceTracker.isCurrentDestinationClaim(handoff.playerUuid, - handoff.server, handoff.conflictSequence)) { - discardPendingPresenceHandoff(handoff); - } - // A handoff stays unassigned while the destination is inside its snapshot - // cooldown. Presence maintenance will attach it to the next allowed snapshot. - return null; - } - if (handoff != null) { - assignPendingPresenceHandoff(handoff, requestId, now); - } - if (created) { - JsonEnvelope request = VotingPluginWire.presenceSnapshotRequest(server, requestId, backendIncarnationId, - backendStartedAt, now); - globalMessageProxyHandler.sendMessage(server, 1, request); - } - return requestId; - } - - private boolean queuePendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { - if (!isPresenceHandoffValid(handoff, now)) { - return false; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - PendingPresenceHandoff current = pendingPresenceHandoffs.get(handoff.playerUuid); - if (current != null && current.conflictSequence > handoff.conflictSequence) { - return false; - } - handoff.requestId = null; - pendingPresenceHandoffs.put(handoff.playerUuid, handoff); - return true; - } - } - - private boolean isPendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { - if (!isPresenceHandoffValid(handoff, now)) { - return false; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - return pendingPresenceHandoffs.get(handoff.playerUuid) == handoff; - } - } - - private void assignPendingPresenceHandoff(PendingPresenceHandoff handoff, UUID requestId, long now) { - if (requestId == null || !isPresenceHandoffValid(handoff, now)) { - return; - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - if (pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { - handoff.requestId = requestId; - } - } - } - - private boolean isPresenceHandoffValid(PendingPresenceHandoff handoff, long now) { - return handoff != null && handoff.playerUuid != null && handoff.connectionId != null - && handoff.conflictSequence > 0L - && now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS; - } - - private Set completePendingPresenceHandoffs(UUID requestId, String server, UUID backendIncarnationId, - long backendStartedAt, long now) { - List completed = new ArrayList<>(); - Set completedPlayers = new LinkedHashSet<>(); - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - PendingPresenceHandoff handoff = entry.getValue(); - if (!requestId.equals(handoff.requestId)) { - return false; - } - if (handoff.server.equalsIgnoreCase(server) - && handoff.backendIncarnationId.equals(backendIncarnationId) - && handoff.backendStartedAt == backendStartedAt) { - completed.add(handoff); - } - return true; - }); - } - for (PendingPresenceHandoff handoff : completed) { - PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(handoff.playerUuid).orElse(null); - if (presence != null && presence.getServer().equalsIgnoreCase(handoff.server) - && presence.getConnectionId().equals(handoff.connectionId)) { - login(handoff.playerName, handoff.uuid, handoff.server); - completedPlayers.add(handoff.playerUuid); - } - releaseDestinationClaim(handoff); - } - return completedPlayers; - } - - /** - * Drains voter-keyed cached rewards when a complete recovery snapshot first - * confirms a player on a dedicated voting proxy. Cross-backend handoffs are - * already processed by their token-bound completion path and are excluded to - * avoid a second login callback. - */ - protected void processDedicatedSnapshotLogins(String server, Set handoffPlayers) { - if (!isDedicatedVotingProxyEnabled() || server == null || server.isBlank()) { - return; - } - Set excluded = handoffPlayers == null ? Collections.emptySet() : handoffPlayers; - for (PlayerPresence presence : backendPlayerPresenceTracker.getOnlinePlayers()) { - if (presence.getServer().equalsIgnoreCase(server) && !excluded.contains(presence.getUuid())) { - login(presence.getPlayerName(), presence.getUuid().toString(), presence.getServer()); - } - } - } - - private void discardPendingPresenceHandoff(String uuid) { - try { - UUID playerUuid = UUID.fromString(uuid.trim()); - PendingPresenceHandoff removed; - synchronized (pendingPresenceHandoffs) { - removed = pendingPresenceHandoffs.remove(playerUuid); - } - releaseDestinationClaim(removed); - } catch (Exception ignored) { - // Invalid identities are rejected by the presence tracker. - } - } - - private void discardPendingPresenceHandoff(PendingPresenceHandoff handoff) { - boolean removed = false; - synchronized (pendingPresenceHandoffs) { - if (handoff != null && pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { - pendingPresenceHandoffs.remove(handoff.playerUuid); - removed = true; - } - } - if (removed) { - releaseDestinationClaim(handoff); - } - } - - private void discardPendingPresenceHandoffs(String server) { - synchronized (pendingPresenceHandoffs) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - if (!entry.getValue().server.equalsIgnoreCase(server)) { - return false; - } - releaseDestinationClaim(entry.getValue()); - return true; - }); - } - } - - private void discardPendingPresenceHandoffs(UUID requestId) { - if (requestId == null) { - return; - } - synchronized (pendingPresenceHandoffs) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - if (!requestId.equals(entry.getValue().requestId)) { - return false; - } - releaseDestinationClaim(entry.getValue()); - return true; - }); - } - } - - private void prunePendingPresenceHandoffs(long now) { - pendingPresenceHandoffs.entrySet().removeIf(entry -> { - PendingPresenceHandoff handoff = entry.getValue(); - if (now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS) { - return false; - } - releaseDestinationClaim(handoff); - return true; - }); - } - - private void releaseDestinationClaim(PendingPresenceHandoff handoff) { - if (handoff != null) { - backendPlayerPresenceTracker.releaseDestinationClaim(handoff.playerUuid, handoff.server, - handoff.conflictSequence); - } - } - - protected void retryPendingPresenceHandoffs(long now) { - List retry = new ArrayList<>(); - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - for (PendingPresenceHandoff handoff : pendingPresenceHandoffs.values()) { - UUID activeRequestId = backendPlayerPresenceTracker.getPendingSnapshotRequestId(handoff.server, now); - if (handoff.requestId != null && !handoff.requestId.equals(activeRequestId)) { - handoff.requestId = null; - } - if (handoff.requestId == null) { - retry.add(handoff); - } - } - } - for (PendingPresenceHandoff handoff : retry) { - requestBackendPresenceSnapshot(handoff.server, handoff, now, true); - } - } - - protected int getPendingPresenceHandoffCount() { - synchronized (pendingPresenceHandoffs) { - return pendingPresenceHandoffs.size(); - } - } - - protected void scheduleBackendPresenceStartupResync() { - ScheduledExecutorService scheduler = getScheduler(); - if (method == null || !method.supportsBackendPresence() || scheduler == null) { - return; - } - scheduler.schedule(this::requestBackendPresenceStartupResync, - PRESENCE_STARTUP_RESYNC_DELAY_SECONDS, TimeUnit.SECONDS); - } - - protected void requestBackendPresenceStartupResync() { - if (!enabled || method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null) { - return; - } - long requestedAt = System.currentTimeMillis(); - int delay = 1; - for (String server : getAllAvailableServers()) { - if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_RESYNC_REQUEST)) { - continue; - } - globalMessageProxyHandler.sendMessage(server, delay++, - VotingPluginWire.presenceResyncRequest(server, UUID.randomUUID(), requestedAt)); - } - } - - private void maintainBackendPresence() { - if (!enabled || method == null || !method.supportsBackendPresence()) { - return; - } - expireBackendPresence(PRESENCE_BACKEND_TIMEOUT_MILLIS); - for (String server : getAllAvailableServers()) { - if (pendingBackendRecoverySnapshots.contains(presenceServerKey(server))) { - requestBackendPresenceSnapshot(server); - } - } - retryPendingPresenceHandoffs(System.currentTimeMillis()); - } - - private String presenceServerKey(String server) { - return server == null ? "" : server.trim().toLowerCase(java.util.Locale.ROOT); - } - - private boolean isPresenceServerValid(String server, String subChannel) { - // The presence protocol's trust boundary is the configured backend set. The - // selected transport must only be accessible to backend servers trusted not to - // impersonate one another. - if (server == null || server.isBlank() || !isServerValid(server)) { - debug("Ignored " + subChannel + " presence envelope for an unconfigured server"); - return false; - } - return true; - } - - private boolean isPresenceGenerationValid(UUID backendIncarnationId, long backendStartedAt, - long presenceTimestamp, String subChannel) { - if (backendIncarnationId == null || backendStartedAt <= 0L || presenceTimestamp < backendStartedAt) { - debug("Ignored " + subChannel + " presence envelope with an invalid backend generation"); - return false; - } - return true; - } - - /** - * Removes presence owned by backends that have stopped reporting heartbeats. - * Scheduling and timeout configuration are intentionally left to dedicated - * proxy mode. - * - * @param timeoutMillis maximum backend silence before expiry - * @return expired backend server names - */ - public Set expireBackendPresence(long timeoutMillis) { - if (method == null || !method.supportsBackendPresence()) { - return Collections.emptySet(); - } - long now = System.currentTimeMillis(); - Set expired = backendPlayerPresenceTracker.expireBackends(now, timeoutMillis); - for (String server : expired) { - discardPendingPresenceHandoffs(server); - // Keep recovery pending while this generation is unavailable. If the same - // backend process resumes, its heartbeat can mark it available again and the - // maintenance task will request a fresh snapshot of players who stayed online. - pendingBackendRecoverySnapshots.add(presenceServerKey(server)); - } - synchronized (pendingPresenceHandoffs) { - prunePendingPresenceHandoffs(now); - } - return expired; - } - - public void login(String playerName, String uuid, String serverName) { - if (!getConfig().getOnlineMode()) { - uuid = getUUID(playerName); - } - - try { - if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { - uuid = UUID.fromString(uuid.trim()).toString(); - } - } catch (Exception ignored) { - // ignore - } - - if (getConfig().getOnlineMode()) { - addNonVotedPlayer(uuid, playerName); - } - if (isPlayerOnlineForVoteRouting(playerName)) { - if (getConfig().getGlobalDataEnabled()) { - if (getGlobalDataHandler().isTimeChangedHappened()) { - getGlobalDataHandler().checkForFinishedTimeChanges(); - } - } - - checkCachedVotes(serverName); - retryPendingOnlineBroadcasts(serverName); - retryPendingTimeBroadcasts(serverName); - checkOnlineVotes(playerName, uuid, serverName); - multiProxyHandler.login(uuid, playerName); - } - } - - private void logInfo(String msg) { - log(msg); - } - - public abstract void logSevere(String message); - - public void onDisable() { - stopControlServices(false); - getVoteCacheHandler().saveVoteCache(); - - if (getProxyMysqlMessenger() != null) { - getProxyMysqlMessenger().shutdown(); - } - - if (getProxyMySQL() != null) { - getProxyMySQL().shutdown(); - } - if (multiProxyHandler != null) { - multiProxyHandler.close(); - } - - if (socketHandler != null) { - socketHandler.closeConnection(); - } - - if (redisHandler != null) { - redisHandler.close(); - } - if (redisPublisherPool != null) { - redisPublisherPool.close(); - redisPublisherPool = null; - } - - bungeeTimeChecker.shutdown(); - - if (getGlobalDataHandler() != null) { - getGlobalDataHandler().shutdown(); - } - - enabled = false; - } - - public void onPluginMessageReceived(DataInputStream in) { - runAsync(() -> { - try { - final String headerSub; - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - headerSub = encryptionHandler.decrypt(in.readUTF()); - } else { - headerSub = in.readUTF(); - } - - int size = in.readInt(); // sanity only - - if (getConfig().getDebug()) { - debug("Received plugin message header=" + headerSub + " size=" + size); - } - - String payload = ""; - if (size > 0) { - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - payload = encryptionHandler.decrypt(in.readUTF()); - } else { - payload = in.readUTF(); - } - } - - JsonEnvelope envelope = JsonEnvelopeCodec.decode(payload); - - if (!headerSub.equalsIgnoreCase(envelope.getSubChannel())) { - if (getConfig().getDebug()) { - warn("PluginMessage subChannel mismatch: header=" + headerSub + " env=" - + envelope.getSubChannel()); - } - return; - } - - globalMessageProxyHandler.onMessage(envelope); - } catch (Exception e) { - e.printStackTrace(); - } - }); - } - - private UUID parseUUIDFromString(String uuidAsString) { - String[] parts = { "0x" + uuidAsString.substring(0, 8), "0x" + uuidAsString.substring(8, 12), - "0x" + uuidAsString.substring(12, 16), "0x" + uuidAsString.substring(16, 20), - "0x" + uuidAsString.substring(20, 32) }; - - long mostSigBits = Long.decode(parts[0]).longValue(); - mostSigBits <<= 16; - mostSigBits |= Long.decode(parts[1]).longValue(); - mostSigBits <<= 16; - mostSigBits |= Long.decode(parts[2]).longValue(); - - long leastSigBits = Long.decode(parts[3]).longValue(); - leastSigBits <<= 48; - leastSigBits |= Long.decode(parts[4]).longValue(); - - return new UUID(mostSigBits, leastSigBits); - } - - public synchronized void processQueue() { - while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { - VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); - if (!vote.isProcessed()) { - VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null - : VoteTotalsSnapshot.parseStorage(vote.getTotals()); - QueuedVoteResult result = vote(vote.getName(), vote.getService(), true, false, vote.getTime(), queuedTotals, - vote.getUuid(), vote); - if (result == QueuedVoteResult.RETRY) { - scheduleTimeVoteRetry(); - return; - } - if (result == QueuedVoteResult.TERMINAL) { - warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/" - + ServiceSiteValidator.sanitizeForLog(vote.getService())); - } - } - if (!getVoteCacheHandler().removeTimeVote(vote)) { - scheduleTimeVoteRetry(); - return; - } - } - } - - private void scheduleTimeVoteRetry() { - if (timeVoteRetryScheduled || getScheduler() == null) { - return; - } - timeVoteRetryScheduled = true; - try { - getScheduler().schedule(() -> { - synchronized (VotingPluginProxy.this) { - timeVoteRetryScheduled = false; - } - processQueue(); - }, 5, TimeUnit.SECONDS); - } catch (RuntimeException e) { - timeVoteRetryScheduled = false; - debug("Unable to schedule rollover vote retry: " + e.getMessage()); - } - } - - public void reload() { - method = BungeeMethod.getByName(getConfig().getBungeeMethod()); - if (getMethod() == null) { - method = BungeeMethod.PLUGINMESSAGING; - } - warnUnsupportedDedicatedVotingProxyMode(); - - setCurrentVotePartyVotesRequired( - getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); - loadMultiProxySupport(); - startControlServices(); - } - - private void warnUnsupportedDedicatedVotingProxyMode() { - if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { - logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, or SOCKETS; PLUGINMESSAGING is disabled for " - + "dedicated-proxy routing. Falling back to normal proxy routing."); - } - } - - public abstract void runAsync(Runnable run); - - /** Platform name used only for the transport-neutral Control discovery contract. */ - public abstract String getProxyPlatform(); - - public abstract void runConsoleCommand(String command); - - public abstract void saveVoteCacheFile(); - - public abstract void reloadCore(boolean mysql); - - public abstract boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue); - - private static final int PLUGIN_MESSAGE_HARD_LIMIT = 32767; - private static final int PLUGIN_MESSAGE_SOFT_LIMIT = 30000; - - public void sendPluginMessageServer(String server, int delay, JsonEnvelope envelope) { - getScheduler().schedule(() -> sendPluginMessageServerNow(server, envelope), delay * 5L, TimeUnit.MILLISECONDS); - } - - /** - * Sends a plugin-message envelope immediately and reports whether the proxy - * accepted it for delivery. - * - * @param server target backend server - * @param envelope envelope to send - * @return true when the proxy accepted the message for delivery - */ - protected boolean sendPluginMessageServerNow(String server, JsonEnvelope envelope) { - final String subChannel = envelope.getSubChannel(); - final String payload = JsonEnvelopeCodec.encode(envelope); - - final byte[] subChannelBytes = subChannel.getBytes(java.nio.charset.StandardCharsets.UTF_8); - final byte[] payloadBytes = payload.getBytes(java.nio.charset.StandardCharsets.UTF_8); - - // Estimate bytes written: - // - writeUTF adds 2-byte length prefix + UTF-8 bytes - // - writeInt is 4 bytes - int estimatedSize = 2 + subChannelBytes.length + // subChannel UTF (len prefix + bytes) - 4 + // payload length int - 2 + payloadBytes.length; // payload UTF (len prefix + bytes) - - if (estimatedSize > PLUGIN_MESSAGE_SOFT_LIMIT) { - debug("[PluginMessage] Payload nearing limit (" + estimatedSize + " bytes) server=" + server - + " subChannel=" + subChannel + " — consider Redis instead"); - } - - if (estimatedSize > PLUGIN_MESSAGE_HARD_LIMIT) { - debug("[PluginMessage] Payload TOO LARGE (" + estimatedSize + " bytes, max=" + PLUGIN_MESSAGE_HARD_LIMIT - + ") server=" + server + " subChannel=" + subChannel + " — NOT sent"); - return false; - } - - try (ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); - DataOutputStream out = new DataOutputStream(byteOutStream)) { - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - out.writeUTF(encryptionHandler.encrypt(subChannel)); - } else { - out.writeUTF(subChannel); - } - - // sanity only: MUST be bytes, not chars - out.writeInt(payloadBytes.length); - - if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { - out.writeUTF(encryptionHandler.encrypt(payload)); - } else { - out.writeUTF(payload); - } - out.flush(); - - boolean sent = sendPluginMessageData(server, getConfig().getPluginMessageChannel().toLowerCase(), - byteOutStream.toByteArray(), false); - if (getConfig().getDebug()) { - debug((sent ? "Sent" : "Could not send") + " plugin envelope (" + estimatedSize + " bytes) " + server - + " " + subChannel + " " + envelope.getFields()); - } - return sent; - } catch (Exception e) { - e.printStackTrace(); - return false; - } - } - - private DefaultJedisClientConfig buildRedisClientConfig() { - DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder() - .database(getConfig().getRedisDbIndex()).connectionTimeoutMillis(2000).socketTimeoutMillis(2000); - if (getConfig().getRedisUsername() != null && !getConfig().getRedisUsername().isEmpty()) { - config.user(getConfig().getRedisUsername()); - } - if (getConfig().getRedisPassword() != null && !getConfig().getRedisPassword().isEmpty()) { - config.password(getConfig().getRedisPassword()); - } - return config.build(); - } - - public boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope) { - return sendRedisEnvelopeServer(server, envelope, false); - } - - private boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope, boolean useRetryCooldown) { - JedisPool publisherPool = redisPublisherPool; - if (publisherPool == null || (useRetryCooldown && System.currentTimeMillis() < redisPublisherRetryAfter)) { - return false; - } - - try (Jedis jedis = publisherPool.getResource()) { - String channel = getConfig().getRedisPrefix() + "VotingPlugin_" + server; - long subscribers = jedis.publish(channel, JsonEnvelopeCodec.encode(envelope)); - redisPublisherRetryAfter = 0L; - return subscribers > 0; - } catch (Exception e) { - if (useRetryCooldown) { - // Standalone broadcasts remain queued, so their retries can be throttled safely. - redisPublisherRetryAfter = System.currentTimeMillis() + 2000L; - } - debug(e.getMessage()); - return false; - } - } - - public boolean sendMqttEnvelopeServer(String server, JsonEnvelope envelope) { - if (mqttHandler == null) { - return false; - } - try { - mqttHandler.publishEnvelope(getConfig().getMqttPrefix() + "votingplugin/servers/" + server, envelope); - return true; - } catch (Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - return false; - } - } - - public boolean sendSocketEnvelopeServer(String server, JsonEnvelope envelope) { - Map configuration = getConfig().getSpigotServerConfiguration(server); - if (configuration == null) { - return false; - } - String host = configuration.get("Host") instanceof String ? (String) configuration.get("Host") : ""; - int port = configuration.get("Port") instanceof Number ? ((Number) configuration.get("Port")).intValue() : 1298; - if (host.isEmpty()) { - return false; - } - - String payload = JsonEnvelopeCodec.encode(envelope); - String encoded = encryptionHandler != null ? encryptionHandler.encrypt(payload) : payload; - try (Socket socket = new Socket()) { - socket.connect(new InetSocketAddress(host, port), 2000); - try (DataOutputStream output = new DataOutputStream(socket.getOutputStream())) { - output.writeUTF(encoded); - output.flush(); - } - return true; - } catch (Exception e) { - debug(e.getMessage()); - return false; - } - } - - public void sendServerNameMessage() { - for (String s : getAllAvailableServers()) { - sendPluginMessageServer(s, 1, VotingPluginWire.serverName(s)); - } - } - - public void sendVoteParty(String server) { - if (isSomeoneOnlineServerForVoteRouting(server)) { - globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee()); - } - } - - public void setCurrentVotePartyVotes(int amount) { - votePartyVotes = amount; - setVoteCacheVotePartyCurrentVotes(amount); - debug("Current vote party total: " + votePartyVotes); - } - - public abstract void setVoteCacheLastUpdated(); - - public abstract void setVoteCachePrevDay(int day); - - public abstract void setVoteCachePrevMonth(String text); - - public abstract void setVoteCachePrevWeek(int week); - - public abstract void setVoteCacheVoteCacheIgnoreTime(boolean ignore); - - public abstract void setVoteCacheVotePartyCurrentVotes(int votes); - - public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes); - - public void status() { - for (String s : getAllAvailableServers()) { - if (!isSomeoneOnlineServerForVoteRouting(s)) { - log("No players on server " + s + " to send test status message, please retest with someone online"); - } else { - log("Sending request for status message on " + s); - globalMessageProxyHandler.sendMessage(s, 1, VotingPluginWire.status(s)); - } - } - } - - private void sendVoteDelayRejected(String player, String uuid, String service, boolean playerOnline, - String playerServer) { - if (!playerOnline || playerServer == null || !getAllAvailableServers().contains(playerServer)) { - debug("Not sending vote delay rejection for " + player + " because the player is offline"); - return; - } - - globalMessageProxyHandler.sendMessage(playerServer, 1, - VotingPluginWire.voteDelayRejected(player, uuid, service, true)); - } - - public String getWaitUntilDelaySiteFromService(String service) { - for (String site : getConfig().getWaitUntilVoteDelaySites()) { - if (getConfig().getWaitUntilVoteDelayService(site).equalsIgnoreCase(service)) { - return site; - } - } - return ""; - } - - private long getLastVotesTime(String uuid, ArrayList cols, String site, String service, String player, - boolean includeTimeChangeQueue) { - long mostRecentTime = 0; - - if (getVoteCacheHandler().hasOnlineVotes(uuid)) { - ArrayList onlineVotes = getVoteCacheHandler().getOnlineVotes(uuid); - for (OfflineBungeeVote vote : onlineVotes) { - if (vote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, vote.getTime()); - } - } - } - - for (String server : getAllAvailableServers()) { - for (OfflineBungeeVote vote : getVoteCacheHandler().getVotes(server)) { - if (vote.getUuid().equals(uuid) && vote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, vote.getTime()); - } - } - } - - if (includeTimeChangeQueue && player != null) { - for (VoteTimeQueue queuedVote : getVoteCacheHandler().getTimeChangeQueue()) { - if (queuedVote.getName().equalsIgnoreCase(player) - && queuedVote.getService().equalsIgnoreCase(service)) { - mostRecentTime = Math.max(mostRecentTime, queuedVote.getTime()); - } - } - } - - for (Column d : cols) { - if (d.getName().equalsIgnoreCase("LastVotes")) { - DataValue value = d.getValue(); - String[] list = value.getString().split("%line%"); - for (String str : list) { - String[] data = str.split("//"); - if (data[0].equalsIgnoreCase(site)) { - mostRecentTime = Math.max(mostRecentTime, Long.valueOf(data[1])); - } - } - } - } - return mostRecentTime; - } - - public boolean checkVoteDelay(String uuid, String service, ArrayList data) { - return checkVoteDelay(uuid, null, service, data, false); - } - - /** - * Checks the configured vote delay, optionally including accepted votes waiting - * for a GlobalData time change to finish. - * - * @param uuid player UUID - * @param player player name used by the time-change queue - * @param service vote service - * @param data current player data - * @param includeTimeChangeQueue whether queued votes reserve their delay slot - * @return true when the vote may be accepted - */ - public boolean checkVoteDelay(String uuid, String player, String service, ArrayList data, - boolean includeTimeChangeQueue) { - String site = getWaitUntilDelaySiteFromService(service); - if (site.isEmpty()) { - debug("No service site set for " + service + ", skipping vote delay check"); - return true; - } - - int voteDelay = getConfig().getWaitUntilVoteDelayVoteDelay(site); - int voteDelayMin = getConfig().getWaitUntilVoteDelayVoteDelayMin(site); - - long lastVote = getLastVotesTime(uuid, data, site, service, player, includeTimeChangeQueue); - if (lastVote == 0) { - debug("No last vote time found for " + uuid + "/" + service + ", skipping vote delay check"); - return true; - } - - try { - LocalDateTime now = getBungeeTimeChecker().getTime(); - LocalDateTime lastVoteTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastVote), ZoneId.systemDefault()) - .plusHours(getConfig().getTimeHourOffSet()); - - if (!getConfig().getWaitUntilVoteDelayVoteDelayDaily(site)) { - if (voteDelay == 0 && voteDelayMin == 0) { - debug("Vote delay is 0 for " + site + ", skipping vote delay check"); - return true; - } - - LocalDateTime nextvote = lastVoteTime.plusHours((long) voteDelay).plusMinutes((long) voteDelayMin); - return now.isAfter(nextvote); - } - LocalDateTime resetTime = lastVoteTime.withHour(getConfig().getWaitUntilVoteDelayVoteDelayHour(site)) - .withMinute(0).withSecond(0); - LocalDateTime resetTimeTomorrow = resetTime.plusHours(24); - - if (lastVoteTime.isBefore(resetTime)) { - if (now.isAfter(resetTime)) { - debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); - return true; - } - } else { - if (now.isAfter(resetTimeTomorrow)) { - debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); - return true; - } - } - } catch (Exception e) { - e.printStackTrace(); - } - - debug("Vote delay is not met for " + uuid + "/" + service + ", skipping vote"); - return false; - } - - public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid) { - vote(player, service, realVote, timeQueue, queueTime, text, uuid, null); - } - - private enum QueuedVoteResult { - SUCCESS, RETRY, TERMINAL - } - - private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, - VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote) { - try { - if (!ServiceSiteValidator.isValid(service)) { - warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'"); - return QueuedVoteResult.TERMINAL; - } - if (!MinecraftUsernameValidator.isValid(player, getConfig().getBedrockPlayerPrefix())) { - warn("Rejected vote with invalid Minecraft username '" - + MinecraftUsernameValidator.sanitizeForLog(player) + "' from service '" - + MinecraftUsernameValidator.sanitizeForLog(service) + "'"); - return QueuedVoteResult.TERMINAL; - } - - UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); - if (voteId == null) { - voteId = UUID.randomUUID(); - } - - // UUID resolution - if (!getConfig().getOnlineMode()) { - uuid = getUUID(player); - } - - if (uuid == null || uuid.isEmpty()) { - uuid = getUUID(player); - - // Bedrock prefix auto-detect - if (uuid.isEmpty() && !getConfig().getBedrockPlayerPrefix().isEmpty() - && !player.startsWith(getConfig().getBedrockPlayerPrefix())) { - String uuid1 = getUUID(getConfig().getBedrockPlayerPrefix() + player); - if (!uuid1.isEmpty()) { - debug("Detected bedrock player without prefix, adjusting..."); - player = getConfig().getBedrockPlayerPrefix() + player; - uuid = uuid1; - } - } - } - - if (uuid.isEmpty()) { - if (player.startsWith(getConfig().getBedrockPlayerPrefix())) { - log("Ignoring vote since unable to get UUID of bedrock player"); - return QueuedVoteResult.TERMINAL; - } - if (!getConfig().getAllowUnJoined()) { - log("Ignoring vote from " + player + " since player hasn't joined before"); - return QueuedVoteResult.TERMINAL; - } - if (!getConfig().getUUIDLookup()) { - log("Failed to get uuid for " + player); - return QueuedVoteResult.TERMINAL; - } - - debug("Fetching UUID online, since allowunjoined is enabled"); - UUID u = null; - try { - if (getConfig().getOnlineMode()) { - u = fetchUUID(player); - } - } catch (Exception e) { - if (getConfig().getDebug()) { - e.printStackTrace(); - } - } - if (u == null) { - debug("Failed to get uuid for " + player); - return QueuedVoteResult.TERMINAL; - } - uuid = u.toString(); - } - - // Normalize UUID string if possible - try { - if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { - uuid = UUID.fromString(uuid.trim()).toString(); - } - } catch (Exception ignored) { - // ignore - } - - player = getProperName(uuid, player); - - // Cache online state/server once (IMPORTANT for broadcast logic correctness) - final boolean playerOnline = isPlayerOnlineForVoteRouting(player); - final String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(player) : null; - long time = queueTime != 0 ? queueTime - : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); - - Set broadcastTargets = queuedVote == null ? new LinkedHashSet<>() - : new LinkedHashSet<>(queuedVote.getBroadcastTargets()); - Set broadcastForwardedServers = queuedVote == null ? new LinkedHashSet<>() - : new LinkedHashSet<>(queuedVote.getBroadcastForwardedServers()); - boolean proxyBroadcastHandled = queuedVote != null && queuedVote.isProxyBroadcastHandled(); - boolean processesTotals = getConfig().getPrimaryServer() || !getConfig().getMultiProxySupport(); - boolean managesTotals = processesTotals && getConfig().getBungeeManageTotals(); - boolean canValidateStandaloneBroadcast = canForwardStandaloneBroadcast(managesTotals); - ArrayList data = null; - boolean queueForTimeChange = false; - - // A completion callback can wipe totals and replay older queued votes. Run it - // before loading this vote's database snapshot so the calculations below use - // the post-rollover state. - if (getConfig().getGlobalDataEnabled() && getGlobalDataHandler().isTimeChangedHappened()) { - getGlobalDataHandler().checkForFinishedTimeChanges(); - queueForTimeChange = timeQueue && getGlobalDataHandler().isTimeChangedHappened(); - } - - // Validate the vote before any immediate announcement. This keeps duplicate - // votes rejected by the delay check out of the GlobalData rollover queue and - // prevents announcing a vote that will not be processed. - if (managesTotals) { - if (getProxyMySQL() == null) { - logSevere("Mysql is not loaded correctly, stopping vote processing"); - return QueuedVoteResult.RETRY; - } - - if (!getProxyMySQL().containsKeyQuery(uuid)) { - getProxyMySQL().update(uuid, "PlayerName", new DataValueString(player)); - getProxyMySQL().getUuids().add(uuid); - } - - data = getProxyMySQL().getExactQuery(new Column("uuid", new DataValueString(uuid))); - if (!checkVoteDelay(uuid, player, service, data, queuedVote == null)) { - log("Vote delay is not met for " + player + "/" + service + ", skipping vote"); - sendVoteDelayRejected(player, uuid, service, playerOnline, playerServer); - return QueuedVoteResult.TERMINAL; - } - } - - // Forward an accepted offline broadcast before the still-active GlobalData - // change queues the reward/totals work. The queued delivery state prevents - // replaying broadcasts that already reached a backend. - if (queueForTimeChange) { - VoteTotalsSnapshot projectedTotals = managesTotals ? getProjectedRolloverTotals(data, player) : text; - if (canValidateStandaloneBroadcast && proxyBroadcastDecider.usesImmediateForwarding(playerOnline)) { - broadcastTargets.addAll(proxyBroadcastDecider.resolveTargets(false, null)); - proxyBroadcastHandled = true; - } - VoteTimeQueue delayedVote = new VoteTimeQueue(voteId, player, service, time, - proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, - projectedTotals == null ? "" : projectedTotals.toString(), false, uuid); - if (!getVoteCacheHandler().addTimeVoteToCache(delayedVote)) { - logSevere("Unable to persist queued rollover vote for " + player + "/" + service - + "; skipping proxy broadcast"); - return QueuedVoteResult.RETRY; - } - if (proxyBroadcastHandled) { - for (String target : broadcastTargets) { - Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player, - service, time, projectedTotals == null ? "" : projectedTotals.toString(), false); - if (delayedVote.getBroadcastForwardedServers().addAll(forwarded)) { - broadcastForwardedServers.addAll(forwarded); - persistTimeVoteDelivery(delayedVote); - } - } - } - log("Caching vote from " + player + "/" + service - + " because time change is happening right now"); - return QueuedVoteResult.SUCCESS; - } - - addVoteParty(); - - // Totals processing (primary server OR no multiproxy) - if (processesTotals) { - if (managesTotals) { - int allTimeTotal = getValue(data, "AllTimeTotal", 1); - int monthTotal = getValue(data, "MonthTotal", 1); - - int dateMonthTotal = -1; - if (getConfig().getStoreMonthTotalsWithDate()) { - if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { - dateMonthTotal = getValue(data, getMonthTotalsWithDatePath(), 1); - } else { - dateMonthTotal = monthTotal; - } - } - - int weeklyTotal = getValue(data, "WeeklyTotal", 1); - int dailyTotal = getValue(data, "DailyTotal", 1); - int points = getValue(data, "Points", getConfig().getPointsOnVote()); - - int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); - if (maxVotes > 0) { - LocalDateTime cTime = getBungeeTimeChecker().getTime(); - int days = cTime.getDayOfMonth(); - if (monthTotal > days * maxVotes) { - monthTotal = days * maxVotes; - } - } - - if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { - points = getConfig().getLimitVotePoints(); - } - - text = new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, - votePartyVotes, currentVotePartyVotesRequired, dateMonthTotal); - - ArrayList update = new ArrayList<>(); - update.add(new Column("AllTimeTotal", new DataValueInt(allTimeTotal))); - update.add(new Column("MonthTotal", new DataValueInt(monthTotal))); - if (getConfig().getStoreMonthTotalsWithDate()) { - update.add(new Column(getMonthTotalsWithDatePath(), new DataValueInt(dateMonthTotal))); - } - update.add(new Column("WeeklyTotal", new DataValueInt(weeklyTotal))); - update.add(new Column("DailyTotal", new DataValueInt(dailyTotal))); - update.add(new Column("Points", new DataValueInt(points))); - - debug("Setting totals " + text.toString() + ", voteId=" + voteId + " for " + player + "/" - + service); - getProxyMySQL().update(uuid, update); - } else { - text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); - } - } - if (text == null) { - text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); - } - - VoteLogStatus voteStatus = VoteLogStatus.IMMEDIATE; - boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled - || proxyBroadcastDecider.usesImmediateForwarding(playerOnline)); - Set proxyBroadcastTargets = Collections.emptySet(); - if (standaloneProxyBroadcast) { - // A handled queued broadcast was necessarily sampled while the player was - // offline. Retry only targets that did not previously accept delivery. - proxyBroadcastTargets = proxyBroadcastHandled ? new LinkedHashSet<>(broadcastTargets) - : proxyBroadcastDecider.resolveTargets(false, null); - Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); - remainingTargets.removeAll(broadcastForwardedServers); - broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time, - text == null ? "" : text.toString(), false)); - } - - // =========================== - // Send vote(s) to backend(s) - // =========================== - if (getConfig().getSendVotesToAllServers()) { - for (String s : getAllAvailableServers()) { - - boolean forceCache = getConfig().getWaitForUserOnline() - && (!playerOnline || playerServer == null || !playerServer.equalsIgnoreCase(s)); - - if (forceCache) { - debug("Forcing vote to cache for server " + s); - } - - if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addServerVote(s, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, - text.toString(), broadcastForwarded, standaloneProxyBroadcast, - proxyBroadcastTargets, broadcastForwardedServers, false)); - debug("Caching vote for " + player + " on " + service + " for " + s); - } else { - boolean broadcastHere = !broadcastForwardedServers.contains(s); - if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { - Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets - : proxyBroadcastDecider.resolveTargets(playerOnline, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); - } - - globalMessageProxyHandler.sendMessage(s, 2, - VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); - } - } - } else { - // Single-server mode: online goes to player server; otherwise queue as "online - // vote" - if (playerOnline && playerServer != null && getAllAvailableServers().contains(playerServer)) { - String server = playerServer; - - boolean broadcastHere = !broadcastForwardedServers.contains(server); - if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { - Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets - : proxyBroadcastDecider.resolveTargets(true, playerServer); - broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); - } - - globalMessageProxyHandler.sendMessage(server, 1, - VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), - voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); - - if (canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() - && !standaloneProxyBroadcast) { - Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); - - int bDelay = 2; - for (String targetServer : targets) { - // avoid double-broadcast on the same server that already got the voteOnline - if (targetServer.equalsIgnoreCase(server)) { - continue; - } - if (getConfig().getBlockedServers().contains(targetServer)) { - continue; - } - - globalMessageProxyHandler.sendMessage(targetServer, bDelay, - VotingPluginWire.voteBroadcast(uuid, player, service, time, - text == null ? "" : text.toString(), true)); - bDelay++; - } - } - - // multiproxy: envelope-only clear vote - if (getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { - multiProxyHandler.sendClearVote(uuid, player); - } - } else { - voteStatus = VoteLogStatus.CACHED; - boolean broadcastForwarded = standaloneProxyBroadcast - && broadcastForwardedServers.containsAll(proxyBroadcastTargets); - getVoteCacheHandler().addOnlineVote(uuid, - new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), - broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, - broadcastForwardedServers, false)); - debug("Caching online vote for " + player + " on " + service); - } - - int delay = 2; - for (String s : getAllAvailableServers()) { - globalMessageProxyHandler.sendMessage(s, delay + 1, VotingPluginWire.voteUpdate(uuid, - votePartyVotes, currentVotePartyVotesRequired, service, time, text.toString())); - delay += 2; - } - } - - // Vote logging - if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) { - voteLogMysqlTable.logVote(voteId, voteStatus, service, uuid, player, time, - getVoteCacheHandler().getProxyCachedTotal(uuid)); - } - - // =========================== - // Multiproxy forwarding - // =========================== - if (getConfig().getMultiProxySupport() && getConfig().getPrimaryServer()) { - if (!getConfig().getMultiProxyOneGlobalReward()) { - debug("Sending global proxy vote envelope"); - multiProxyHandler.sendMultiProxyEnvelope(VotingPluginWire.vote(player, uuid, service, time, false, - realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); - } else { - // Only send to other proxies if the player DID NOT already receive reward on a - // backend - boolean shouldSend = true; - if (playerOnline && playerServer != null) { - if (!getConfig().getBlockedServers().contains(playerServer)) { - shouldSend = false; - } - } - - if (shouldSend) { - debug("Sending global proxy voteonline envelope"); - multiProxyHandler - .sendMultiProxyEnvelope(VotingPluginWire.voteOnline(player, uuid, service, time, false, - realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); - } else { - debug("Not sending global proxy message for voteonline, player already got reward"); - } - } - } - if (queuedVote != null) { - queuedVote.setProcessed(true); - if (!getVoteCacheHandler().updateTimeVote(queuedVote)) { - warn("Unable to persist completed rollover vote " + queuedVote.getVoteId() - + "; attempting durable removal immediately"); - } - } - return QueuedVoteResult.SUCCESS; - } catch (Exception e) { - e.printStackTrace(); - return QueuedVoteResult.RETRY; - } - } - - private static final class PendingPresenceHandoff { - private UUID requestId; - private final UUID playerUuid; - private final String playerName; - private final String uuid; - private final String server; - private final UUID connectionId; - private final UUID backendIncarnationId; - private final long backendStartedAt; - private final long conflictSequence; - private final long createdAt; - - private PendingPresenceHandoff(String playerName, String uuid, String server, UUID connectionId, - UUID backendIncarnationId, long backendStartedAt, long conflictSequence, long createdAt) { - this.playerUuid = parsePlayerUuid(uuid); - this.playerName = playerName; - this.uuid = uuid; - this.server = server; - this.connectionId = connectionId; - this.backendIncarnationId = backendIncarnationId; - this.backendStartedAt = backendStartedAt; - this.conflictSequence = conflictSequence; - this.createdAt = createdAt; - } - - private static UUID parsePlayerUuid(String uuid) { - try { - return UUID.fromString(uuid.trim()); - } catch (Exception ignored) { - return null; - } - } - } - - public abstract void warn(String message); - - public abstract ScheduledExecutorService getScheduler(); -} +Yx-jםi+j[hܢͶ:-jZ.)޳W6vR6&V6FWfFwVv&'Bf'FT'&WGWE7G&VӰ'BfFFWE7G&VӰ'BfFFWGWE7G&VӰ'BffS'BfW6WF㰦'BfWBWE66WDFG&W73'BfWB66WC'BfWBU$'BfWBGGGG6ƖVC'BfWBGGGG&WVW7C'BfWBGGGG&W76S'Bf75W6WF㰦'BfFRGW&F㰦'BfFR7FC'BfFR6FFUFS'BfFRTC'BfFRTfg6WC'BfWF'&Ɨ7C'BfWF6V7F㰦'BfWF6V7F3'BfWF6'BfWFƖVD66WC'BfWFƗ7C'BfWF'BfWF6WC'BfWFUTC'BfWF67W'&VB67W'&VD6'BfWF67W'&VB66VGVVDWV7WF%6W'f6S'BfWF67W'&VBFUVC'B&rV6Ɨ6R6ƖVBGGc2GDW6WF㰠'B6&V6FWGf6VF6&RFRFUGS'B6&V6FWGf6VF6&R'VvVVv&FFv&FFFW%&'B6&V6FWGf6VF6&R'VvVVv&FFv&ו5ð'B6&V6FWGf6VF6&R'VvVVFR'VvVUFT6V6W#'B6&V6FW6VV7'FV7'FFW#'B6&V6FW6V66'6W#'B6&V6FW6V6W'fW&66FV26VfVS'B6&V6FW6V6W'fW&66FV26VfVT6FV3'B6&V6FW6V6W'fW&6v&v&W76vTƗ7FVW#'B6&V6FW6V6W'fW&6v&v&W76vU&FW#'B6&V6FW6V6W'fW&6GBGDFW#'B6&V6FW6V6W'fW&6GBGE6W'fW$6Ӱ'B6&V6FW6V6W'fW&6ח7ו7W76VvW#'B6&V6FW6V6W'fW&6&VF2&VF4FW#'B6&V6FW6V6W'fW&6&VF2&VF4Ɨ7FVW#'B6&V6FW6V6W'fW&666WG26ƖVDFW#'B6&V6FW6V6W'fW&666WG266WDFW#'B6&V6FW6V6W'fW&666WG266WE&V6VfW#'B6&V6FW6V76V㰦'B6&V6FW6V7FFGS'B6&V6FW6V7FFFFfVS'B6&V6FW6V7FFFFfVT&V㰦'B6&V6FW6V7FFFFfVTC'B6&V6FW6V7FFFFfVU7G&s'B6&V6FW6V7ח76frח76fs'B6&V6FWfFwVv&'&F67B&'&F67DFV6FW#'B6&V6FWfFwVv&66RfFT66S'B6&V6FWfFwVv&66RfFT66TFW#'B6&V6FWfFwVv&66RfFVBfFVEW'57F&vS'B6&V6FWfFwVv&66RfFVBfFVEW'466S'B6&V6FWfFwVv&6G&6G&6V7F#'B6&V6FWfFwVv&6G&7FVD6G&vW#'B6&V6FWfFwVv&VF&VF&FW#'B6&V6FWfFwVv&VF&VF&WFC'B6&V6FWfFwVv&VF&VF&6W'fW%66WD6fwW&F㰦'B6&V6FWfFwVv&VF&VF&6W'fW%66WD6fwW&F'VvVS'B6&V6FWfFwVv&&W6V6R&6VEW%&W6V6UG&6W#'B6&V6FWfFwVv&&W6V6RW%&W6V6S'B6&V6FWfFwVvFWVWVRfFUFUVWVS'B6&V6FWfFwVvFfFW"FfFW#'B6&V6FWfFwVvWF֖V7&gEW6W&UfƖFF#'B6&V6FWfFwVvWF6W'f6U6FUfƖFF#'B6&V6FWfFwVvfFVrfFTtח7F&S'B6&V6FWfFwVvfFVrfFTtח7F&RfFTu7FGW3'B6vvRw66VVVC'B6vvRw66&V7C'B&VF26ƖVG2VF2FVfVDVF46ƖVD6fs'B&VF26ƖVG2VF27DE'C'B&VF26ƖVG2VF2VF3'B&VF26ƖVG2VF2VF5ð'B&vWGFW#'B&6WGFW#V&Ɩ2'7G&7B672fFuVv&&fFR7FF2fr$U4T4UDdeDTUEԔĕ2FUVBԔUDU2F֖Ɨ2"&fFR7FF2fr$U4T4U5D%EU$U55DT4T4E2Tð&fFR7FF2fr$U4T4UDT4UDU%d4T4E23ð&fFR7FF2fr$U4T4U$4TEDTUEԔĕ2FUVB4T4E2F֖Ɨ2vWGFW 6WGFW &fFRBfFU'GfFW2vWGFW 6WGFW &fFRB7W'&VEfFU'GfFW5&WV&VBvWGFW 6WGFW &fFR&ח7W6W%F&R&ו5ð&fFRV7'FFW"V7'FFW#&fFR67G&r6ƖVDFW#6ƖVDFW3&fFR66WDFW"66WDFW#vWGFW 6WGFW &fFR&VfFfW$V&VBG'VSvWGFW &fFR67W'&VD6UTB7G&sWVEW$T66RWr67W'&VD6ₓvWGFW 6WGFW &fFRv&FFFW%&v&FFFW#vWGFW &fFR&VF4FW"&VF4FW#&fFRVF5&VF5V&Ɨ6W%ð&fFRfFRr&VF5V&Ɨ6W%&WG'gFW#&fFR&VFUfFU&WG'66VGVVC&fFR&VFUfFTFVƗfW'&WG'66VGVVC&fFR&V66VEfFTFVƗfW'&WG'66VGVVC&fFR&VV&VCvWGFW 6WGFW &fFRVF&FW"VF&FW#vWGFW &fFR'VvVUFT6V6W"'VvVUFT6V6W#vWGFW 6WGFW &fFR'VvVTWFBWFCvWGFW &fFRGDFW"GDFW#vWGFW &fFRv&W76vU&FW"v&W76vU&FW#vWGFW 6WGFW &fFRו7W76VvW"&ח7W76VvW#vWGFW &fFRfFT66TFW"fFT66TFW#vWGFW &fFRfFVEW'466RfFVEW'466SvWGFW &fFRf&6VEW%&W6V6UG&6W"&6VEW%&W6V6UG&6W"Wr&6VEW%&W6V6UG&6W"&fFRfUTBVFu&W6V6TFfcVFu&W6V6TFfg2Wr6ₓ&fFRf6WC7G&sVFt&6VE&V6fW'66G267W'&VD6WtW6WB&fFRfFR6G&6V7F"6G&6V7F#&fFRfFR7FVD6G&vW"7FVD6G&vW#V&Ɩ2fFuVv&V&VBG'VS'VvVUFT6V6W"Wr'VvVUFT6V6W"vWD6frvWEFURvWD6frvWEFTW$fe6WBvWD6frvWEFUvVVfe6WBfW'&FPV&Ɩ2fBFV'Vr7G&rFWBFV'Vs"FWBРfW'&FPV&Ɩ2rvWD7EWFFVB&WGW&vWEfFT66T7EWFFVBРfW'&FPV&Ɩ2BvWE&WdF&WGW&vWEfFT66U&WdFРfW'&FPV&Ɩ27G&rvWE&WdF&WGW&vWEfFT66U&WdFРfW'&FPV&Ɩ2BvWE&WevVV&WGW&vWEfFT66U&WevVVРfW'&FPV&Ɩ2fBf7G&rFWBrFWBРfW'&FPV&Ɩ2&V4V&VB&WGW&V&VCРfW'&FPV&Ɩ2&V4v&UFR&WGW&5fFT66Tv&UFRРfW'&FPV&Ɩ2fB6WDv&UFR&Vv&R6WEfFT66UfFT66Tv&UFRv&RРfW'&FPV&Ɩ2fB6WD7EWFFVB6WEfFT66T7EWFFVBРfW'&FPV&Ɩ2fB6WE&WdFBF6WEfFT66U&WdFFРfW'&FPV&Ɩ2fB6WE&WdF7G&rFWB6WEfFT66U&WdFFWBРfW'&FPV&Ɩ2fB6WE&WevVVBvVV6WEfFT66U&WevVVvVVРfW'&FPV&Ɩ2fBFT6vVBFUGRGR&VfR&V&R&V7BbvWD6frvWEfFT66UFRvWEfFT66TFW"6V6fFT66UFRvWD6frvWEfFT66UFRРbvWD6frvWDv&FFV&VBv&$v&FFBV&VBv&rFR6vRWfVB"&WGW&㰠РBFVf"7G&r2vWDf&U6W'fW'2bvWDv&FFFW"vWDv&ח7‚6F4W27G&r7DƖU7G"vWDv&FFFW"vWE7G&r2$7DƖR"r7DƖRG'7DƖRrfVTb7DƖU7G"6F6V&W$f&DW6WFRv&PРb6FFUFRrERTfg6WBUD2F7FBFW6֖ƒ7DƖRcc"67G&rFFfVSFFF6WBWr6ₓFFF6WBWB$7EWFFVB"WrFFfVU7G&r""6FFUFRrERTfg6WBUD2F7FBFW6֖ƒFFF6WBWB$f6VE&6W76r"WrFFfVT&Vf6RFFF6WBWBGRF7G&rWrFFfVT&VG'VRvWDv&FFFW"6WDFF2FFF6WBv&W76vU&FW"6VDW76vR2FVfFuVvv&R'VvVUFT6vRFVV6Rv&%6W'fW""2"6wB&VVƖR&V6VFǒ"РV6Rv&%6W'fW""2"v&FFFW"F6&VC"РРv&FFFW"FT6vRGRРfW'&FPV&Ɩ2fBv&r7G&rFWBv&FWBРӰРV&Ɩ2fBFT6vVDfVB7G&r7'bFUGRGRvWDv&FFFW"6WD&V7'bGRF7G&rf6RvWDv&FFFW"6WD&V7'b$f6VE&6W76r"G'VRvWDv&FFFW"6WD&V7'b%&6W76r"f6RРV&Ɩ2fBFT6vVDf6VBFUGRGRbGRWV2FUGRDvWE&ו5‚66VFFFfFW"FǒvWD6VR$7DFFF"РvWE&ו5‚vT6VFFFfFW"bGRvWD6VRFFGRDTtU"bvWD6frvWDv&FFV&VB&WGW&㰠Рf"7G&r2vWDf&U6W'fW'2vWDv&FFFW"6WD&V2$f&6UWFFR"G'VRvWDv&W76vU&FW"6VDW76vR2fFuVvv&R'VvVUFT6vRР&6W75VWVRР򢠠Bו5v&FFFW"ࠒV&Ɩ2fBDח7„ח76frח76frח76frv&FFח76frbח76frvWD7DR4VGח76frvWDFF&6R4VGu6WfW&R$ו52B6fwW&VB6'&V7Fǒ"$֗76r7BFF&6R7C"ח76frvWD7DR"F#"ח76frvWDFF&6R6WE&ו5†V“&WGW&㰠Р6WE&ו5†Wr&ח7W6W%F&R%fFuVvW6W'2"ח76frvWD6frvWDFV'VrfW'&FPV&Ɩ2fBFV'Vr5W6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBu6WfW&R7G&r7G&rfFuVv&F2u6WfW&R7G&rРfW'&FPV&Ɩ2fBtf7G&r7G&rfFuVv&F2tf7G&rРfW'&FPV&Ɩ2fBFV'VrF&v&RBbvWD6frvWDFV'VrB&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vr7G&r7G"FV'Vs"7G"Рғ'&Ɨ7C7G&s6W'fW'2Wr'&Ɨ7C7G&svWDf&U6W'fW'2bvWD6frvWDv&FFV&VBbvWD6frvWDv&FFW6Tו5‚6WDv&FFFW"Wrv&FFFW%&Wrv&ו5‚%fFuVvv&FF"vWE&ו5‚vWDח7‚fW'&FPV&Ɩ2fBFV'VtWW6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vtr7G&rFWBFV'VrFWBРfW'&FPV&Ɩ2fBf7G&rFWBtfFWBРfW'&FPV&Ɩ2fBu6WfW&R7G&rFWBfFuVv&F2u6WfW&RFWBРfW'&FPV&Ɩ2fBv&r7G&rFWBv&FWBР6W'fW'2fW'&FPV&Ɩ2fBFT6vVDfVB7G&r7'bFUGRGRfFuVv&F2FT6vVDfVB7'bGRРfW'&FPV&Ɩ2fBFT6vVDf6VBFUGRGRfFuVv&F2FT6vVDf6VBGRРғV6R6WDv&FFFW"Wrv&FFFW%&Wrv&ו5‚%fFuVvv&FF"v&FFח76frfW'&FPV&Ɩ2fBFV'VtWW6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vtr7G&rFWBFV'VrFWBРfW'&FPV&Ɩ2fBf7G&rFWBtfFWBРfW'&FPV&Ɩ2fBu6WfW&R7G&rFWBfFuVv&F2u6WfW&RFWBРfW'&FPV&Ɩ2fBv&r7G&rFWBv&FWBР6W'fW'2fW'&FPV&Ɩ2fBFT6vVDfVB7G&r7'bFUGRGRfFuVv&F2FT6vVDfVB7'bGRРfW'&FPV&Ɩ2fBFT6vVDf6VBFUGRGRfFuVv&F2FT6vVDf6VBGRРғРWFFRv&66V6V2V6vVBg&&vvWDv&FFFW"vWDv&ח7‚FW$6VGR$v&UFR"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$D"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR%tTT"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$D"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$f6VE&6W76r"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR%&6W76r"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$f&6UWFFR"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$7EWFFVB"$TDTDUB"Р6VGW2V6vVBg&&vvWE&ו5‚FW$6VGR%FfFW$v&R"%d$4"R"vWE&ו5‚FW$6VGR$6V6v&B"%d$4"R"vWE&ו5‚FW$6VGR%&V֖FVB"%d$4"R"vWE&ו5‚FW$6VGR$F6&T'&F67B"%d$4"R"vWE&ו5‚FW$6VGR$7DƖR"%d$4"#"vWE&ו5‚FW$6VGR%W$R"%d$4"3"vWE&ו5‚FW$6VGR$FǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR%vVVǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$FfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR$&W7DFfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR%vVVfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR$&W7EvVVfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR%fFU'GfFW2"$BDTdTBsr"vWE&ו5‚FW$6VGR$FfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR%G2"$BDTdTBsr"vWE&ו5‚FW$6VGR$vW7DFǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$FUFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$vW7DFǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$FFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$vW7EvVVǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$7DFFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$7EvVVǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$7DFǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$ffƖU&Wv&G2"$TDTDUB"vWE&ו5‚FW$6VGR$FfFU7G&V7EWFFR"$TDTDUB"bvWD6frvWE7F&TFFF5vFFFRvWE&ו5‚FW$6VGRvWDFFF5vFFFUF6FFUFRr$BDTdTBsr"vWE&ו5‚FW$6VGRvWDFFF5vFFFUF6FFUFRrW4F2$BDTdTBsr"vWE&ו5‚FW$6VGRvWDFFF5vFFFUF6FFUFRrW4F2"$BDTdTBsr"РРV&Ɩ2fBFD7W'&VEfFU'GfFW2BVBfFU'GfFW2VC6WEfFT66UfFU'G7W'&VEfFW2fFU'GfFW2FV'Vr$7W'&VBfFR'GFFâ"fFU'GfFW2РV&Ɩ2fBFDfFVEW"7G&rWVB7G&rW$RfFVEW'466RFEW"WVBW$RРV&Ɩ2fBFEfFU'GbvWD6frvWEfFU'GV&VBFD7W'&VEfFU'GfFW26V6fFU'GРРV&Ɩ2'7G&7BfB'&F67B7G&rW76vR&fFR6WC7G&s6VE&'&F67B6WC7G&sF&vWG27G&rWVB7G&rW"7G&r6W'f6RrFR7G&rFWB&Vv4ƖR6WC7G&sf'v&FVBWrƖVD66WCₓf"7G&rF&vWE6W'fW"F&vWG26VfVRVfVRfFuVvv&RfFT'&F67BWVBW"6W'f6RFRFWBv4ƖRb6VE&'&F67DVfVTrF&vWE6W'fW"VfVRf'v&FVBFBF&vWE6W'fW"РР&WGW&f'v&FVCР򢠠6VG27FFR&'&F67BF&VvFR6VV7FVBG&7'BB&W'G0vWFW"FBG&7'B66WFVBFRW76vRࠒ&6W'fW"F&vWB&6VB6W'fW &VfVR7FFR'&F67BVfVP&WGW&G'VRǒvVFRG&7'B66WFVBFRW76vP&FV7FVB&V6VE&'&F67DVfVTr7G&r6W'fW"6VfVRVfVR7vF6WFB66REC&WGW&6VDGDVfVU6W'fW"6W'fW"VfVR66RՕ5àb&ח7W76VvW"V’&WGW&f6SРG'&ח7W76VvW"6VEF&6VB6W'fW"VfVR&WGW&G'VS6F65W6WFRFV'VrRvWDW76vR&WGW&f6SР66RTtU54ts&WGW&6VEVvW76vU6W'fW$r6W'fW"VfVR66R$TD3&WGW&6VE&VF4VfVU6W'fW"6W'fW"VfVRG'VR66R44UE37FFR'&F67G2W6RFR6RFƗVB6ƖVB2&VfVW2F2&W6W'fW2FR66WB6V7FBG2FVƗfW'6vVFvVVB7FVBb7&VFr6V6B6'BƗfVB66WBࠐ6ƖVDFW"66WD6ƖVB6ƖVDFW2VV6ƖVDFW2vWB6W'fW"b66WD6ƖVBV’&WGW&f6SРG'66WD6ƖVB6VDVfVRVfVR&WGW&G'VS6F6'VFTW6WFRFV'VrRvWDW76vR&WGW&f6SРFVfVC&WGW&f6SРРV&Ɩ276&旦VBfB6V666VEfFW27G&r6W'fW"BFVb56W'fW%fƖB6W'fW"b56VTƖU6W'fW$f%fFU&WFr6W'fW"bvWEfFT66TFW"5fFW26W'fW"bbvWD6frvWD&6VE6W'fW'26F26W'fW"'&Ɨ7CffƖT'VvVUfFS2vWEfFT66TFW"vWEfFW26W'fW"'&Ɨ7CffƖT'VvVUfFS&VfVBWr'&Ɨ7Cₓb24VGBVBV&W$efFW226Rf"ffƖT'VvVUfFR66R2b66R4FVƗfW'7FFTF'GbbW'67E6W'fW%fFTFVƗfW'6W'fW"66R6FVSРb66R5&'&F67DFVBbb66RVVG4'&F67D6W'fW"6WC7G&sf'v&FVB6VE&'&F67B6V7F26vWF6W'fW"66RvWEWVB66RvWEW$R66RvWE6W'f6R66RvWEFR66RvWEFWBf6Rb66RvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVB66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRbW'67E6W'fW%fFTFVƗfW'6W'fW"66R6FVSРРР&VF6VBG'VSbvWD6frvWEvDf%W6W$ƖRb5W$ƖTf%fFU&WFr66RvWEW$RF6VBf6SV6Rb5W$ƖTf%fFU&WFr66RvWEW$RbbvWD7W'&VEW%6W'fW$f%fFU&WFr66RvWEW$RWV26W'fW"F6VBf6SРРbF6VB&V'&F67DW&R66RVVG4'&F67D6W'fW"b66R5&'&F67DFVBbb'&F67DW&PbbvWD6frvWE&'&F67DV&VB&VW$ƖR5W$ƖTf%fFU&WFr66RvWEW$R7G&rW%6W'fW"W$ƖRvWD7W'&VEW%6W'fW$f%fFU&WFr66RvWEW$RVð6WC7G&sF&vWG2&'&F67DFV6FW"&W6fUF&vWG2W$ƖRW%6W'fW"'&F67DW&R&'&F67DFV6FW"6VD'&F67B6W'fW"F&vWG2Рv&W76vU&FW"6VDW76vR6W'fW"FVfFuVvv&RfFR66RvWEW$R66RvWEWVB66RvWE6W'f6R66RvWEFRf6R66R5&VfFR66RvWEFWB66RvWEfFTBvWD6frvWD'VvVTvUFF2'&F67DW&RVV&W$efFW2FVVҲ&VfVBFB66RV6RFV'Vr$B6VFrfFR&V6W6RW6W"6wB6W'fW""6W'fW"# 66RF7G&rРРvWEfFT66TFW"&VfU6W'fW%fFW26W'fW"&VfVBV6RFV'Vr$66VBfFW2f"6W'fW#"6W'fW"РV6RFV'Vr$66VBfFW2f"6W'fW#"6W'fW"РРV6RFV'Vr%6W'fW"BfƖC"6W'fW"РРV&Ɩ276&旦VBfB6V6ƖUfFW27G&rW"7G&rWVB7G&r6W'fW"BFVb5W$ƖTf%fFU&WFrW"bbvWEfFT66TFW"4ƖUfFW2WVB'&Ɨ7CffƖT'VvVUfFS2vWEfFT66TFW"vWDƖUfFW2WVBb24VGb6W'fW"V’6W'fW"vWD7W'&VEW%6W'fW$f%fFU&WFrW"РbvWD6frvWD&6VE6W'fW'26F26W'fW"BVBV&W$efFW2B27G&V҂fFW"fFRfFR5&Wv&DFVƗfW&VB6VB&VFVƗfW&VE&Wv&Bf6S'&Ɨ7CffƖT'VvVUfFS&WFVBWr'&Ɨ7Cₓf"ffƖT'VvVUfFR66R2b66R5&'&F67DFVB6WC7G&sVFuF&vWG2WrƖVD66WC66RvWD'&F67EF&vWG2VFuF&vWG2&VfT†66RvWD'&F67Df'v&FVE6W'fW'2Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b&6VE6W'fW'2V’VFuF&vWG2&VfT†&6VE6W'fW'2Р66RvWD'&F67Df'v&FVE6W'fW'2FD‡6VE&'&F67BVFuF&vWG266RvWEWVB66RvWEW$R66RvWE6W'f6R66RvWEFR66RvWEFWBf6R66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRР&V'&F67DW&R66RVVG4'&F67D6W'fW"b66R5&'&F67DFVBbb'&F67DW&PbbvWD6frvWE&'&F67DV&VB7G&rW%6W'fW"6W'fW"V’6W'fW"vWD7W'&VEW%6W'fW$f%fFU&WFrW"6WC7G&sF&vWG2&'&F67DFV6FW"&W6fUF&vWG2G'VRW%6W'fW"'&F67DW&R&'&F67DFV6FW"6VD'&F67B6W'fW"F&vWG2Рb66R5&Wv&DFVƗfW&VBv&W76vU&FW"6VDW76vR6W'fW"FVfFuVvv&RfFTƖR66RvWEW$R66RvWEWVB66RvWE6W'f6R66RvWEFRf6R66R5&VfFR66RvWEFWB66RvWEfFTBvWD6frvWD'VvVTvUFF2'&F67DW&RVV&W$efFW2FR&VfVR26fƖB'&F67BFVƗfW'f"FP7W'&VBF&vWB&V6&BB6&WfW6ǒVFr7FFP&WG'6BV6RFR6RfFRvFW"ࠐb66R5&'&F67DFVBbb'&F67DW&R66RvWD'&F67Df'v&FVE6W'fW'2FB6W'fW"66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRР66R6WE&Wv&DFVƗfW&VBG'VRFVƗfW&VE&Wv&BG'VSFVVҲРb66R5&'&F67DFVBbb66R5&'&F67D6WFR&WFVBFB66RРРvWEfFT66TFW"&VfTƖUfFW2WVBf"ffƖT'VvVUfFRVFr&WFVBvWEfFT66TFW"FDƖUfFRWVBVFrРVF&VfVRǐbFVƗfW&VE&Wv&BbbvWD6frvWDVF&7W'BbbvWD6frvWDVF&Tv&&Wv&BVF&FW"6VD6V%fFRWVBW"РРРРР򢠠&WG&W2fFW"ֶWVB7FFR'&F67G2vVW"W2F&vW@&6VBf&R2VvW76vR6'&W"ࠒ&6W'fW"&6VB6W'fW"FBvVB6'&W &FV7FVB76&旦VBfB&WG'VFtƖT'&F67G27G&r6W'fW"Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b6W'fW"V&6VE6W'fW'2Vbb&6VE6W'fW'26F26W'fW"&WGW&㰠Рf"7G&r66VEWVBvWEfFT66TFW"vWDƖUfFUUTG2f"ffƖT'VvVUfFR66RWr'&Ɨ7CvWEfFT66TFW"vWDƖUfFW266VEWVBb66R4FVƗfW'7FFTF'GbbW'67DƖUfFTFVƗfW'66VEWVB66R6FVSРb66R5&'&F67DFVB66RVVG4'&F67D6W'fW"6FVSР6WC7G&sf'v&FVB6VE&'&F67B6V7F26vWF6W'fW"66RvWEWVB66RvWEW$R66RvWE6W'f6R66RvWEFR66RvWEFWBf6Rb66RvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVB66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRb66R5&Wv&DFVƗfW&VBbb66R5&'&F67D6WFRvWEfFT66TFW"&VfTƖUfFR66VEWVB66RV6RW'67DƖUfFTFVƗfW'66VEWVB66RРРРРР&FV7FVB76&旦VBfB&WG'VFuFT'&F67G27G&r6W'fW"Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b6W'fW"V&6VE6W'fW'2Vbb&6VE6W'fW'26F26W'fW"&WGW&㰠РbvWEfFT66TFW"vWEFT6vUVWVRV’&WGW&㰠Рf"fFUFUVWVRfFRWr'&Ɨ7CvWEfFT66TFW"vWEFT6vUVWVRbfFR4FVƗfW'7FFTF'GbbW'67EFUfFTFVƗfW'fFR6FVSРbfFR5&'&F67DFVBfFRvWEWVB4VGfFRvWD'&F67EF&vWG26F26W'fW"fFRvWD'&F67Df'v&FVE6W'fW'26F26W'fW"6FVSР6WC7G&sf'v&FVB6VE&'&F67B6V7F26vWF6W'fW"fFRvWEWVBfFRvWDRfFRvWE6W'f6RfFRvWEFRfFRvWEFF2f6RbfFRvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVBW'67EFUfFTFVƗfW'fFRРРР򢠠W&F6ǒ&WG&W2WfW'VFrfFW"ֶWVB7FFR'&F67BF20&WV&VBf"'&W"G&7'G2v6R&V6fW'FW2B&GV6RW"vࠒ6'&W"WfVBࠒV&Ɩ276&旦VBfB&WG'VFtƖT'&F67G2f"7G&r66VEWVBWrƖVD66WCvWEfFT66TFW"vWDƖUfFUUTG2f"ffƖT'VvVUfFR66RWr'&Ɨ7CvWEfFT66TFW"vWDƖUfFW266VEWVBb66R4FVƗfW'7FFTF'GbbW'67DƖUfFTFVƗfW'66VEWVB66R6FVSРb66R5&'&F67DFVB66R5&'&F67D6WFR6FVSР6WC7G&sVFuF&vWG2WrƖVD66WC66RvWD'&F67EF&vWG2VFuF&vWG2&VfT†66RvWD'&F67Df'v&FVE6W'fW'2Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b&6VE6W'fW'2V’VFuF&vWG2&VfT†&6VE6W'fW'2Р6WC7G&sf'v&FVB6VE&'&F67BVFuF&vWG266RvWEWVB66RvWEW$R66RvWE6W'f6R66RvWEFR66RvWEFWBf6Rb66RvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVB66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRb66R5&Wv&DFVƗfW&VBbb66R5&'&F67D6WFRvWEfFT66TFW"&VfTƖUfFR66VEWVB66RV6RW'67DƖUfFTFVƗfW'66VEWVB66RРРРР&WG'VFuFT'&F67G2РV&Ɩ276&旦VBfB&WG'VFuFT'&F67G2bvWEfFT66TFW"vWEFT6vUVWVRV’&WGW&㰠Рf"fFUFUVWVRfFRWr'&Ɨ7CvWEfFT66TFW"vWEFT6vUVWVRbfFR4FVƗfW'7FFTF'GbbW'67EFUfFTFVƗfW'fFR6FVSРbfFR5&'&F67DFVBfFRvWEWVB4VG6FVSР6WC7G&sVFuF&vWG2WrƖVD66WCfFRvWD'&F67EF&vWG2VFuF&vWG2&VfT‡fFRvWD'&F67Df'v&FVE6W'fW'2Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b&6VE6W'fW'2V’VFuF&vWG2&VfT†&6VE6W'fW'2Р6WC7G&sf'v&FVB6VE&'&F67BVFuF&vWG2fFRvWEWVBfFRvWDRfFRvWE6W'f6RfFRvWEFRfFRvWEFF2f6RbfFRvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVBW'67EFUfFTFVƗfW'fFRРРР&FV7FVB76&旦VB&VW'67EFUfFTFVƗfW'fFUFUVWVRfFRbvWEfFT66TFW"WFFUFUfFRfFRfFR6WDFVƗfW'7FFTF'Gf6R&WGW&G'VSРfFR6WDFVƗfW'7FFTF'GG'VR66VGVUFUfFTFVƗfW'&WG'&WGW&f6SР&fFRfB66VGVUFUfFTFVƗfW'&WG'bFUfFTFVƗfW'&WG'66VGVVBvWE66VGVW"V’&WGW&㰠РFUfFTFVƗfW'&WG'66VGVVBG'VSG'vWE66VGVW"66VGVR76&旦VBfFuVv&F2FUfFTFVƗfW'&WG'66VGVVBf6SР&WG'VFuFT'&F67G2RFUVB4T4E26F6'VFTW6WFRFUfFTFVƗfW'&WG'66VGVVBf6SFV'Vr%V&RF66VGVRFVB'&F67B7FFR&WG'"RvWDW76vRРР&FV7FVB76&旦VB&VW'67E6W'fW%fFTFVƗfW'7G&r6W'fW"ffƖT'VvVUfFRfFRbvWEfFT66TFW"WFFU6W'fW%fFR6W'fW"fFRfFR6WDFVƗfW'7FFTF'Gf6R&WGW&G'VSРfFR6WDFVƗfW'7FFTF'GG'VR66VGVT66VEfFTFVƗfW'&WG'&WGW&f6SР&FV7FVB76&旦VB&VW'67DƖUfFTFVƗfW'7G&rWVBffƖT'VvVUfFRfFRbvWEfFT66TFW"WFFTƖUfFRWVBfFRfFR6WDFVƗfW'7FFTF'Gf6R&WGW&G'VSРfFR6WDFVƗfW'7FFTF'GG'VR66VGVT66VEfFTFVƗfW'&WG'&WGW&f6SР&fFRfB66VGVT66VEfFTFVƗfW'&WG'b66VEfFTFVƗfW'&WG'66VGVVBvWE66VGVW"V’&WGW&㰠Р66VEfFTFVƗfW'&WG'66VGVVBG'VSG'vWE66VGVW"66VGVR76&旦VBfFuVv&F266VEfFTFVƗfW'&WG'66VGVVBf6SР&WG'66VEfFTFVƗfW'W'67FV6RRFUVB4T4E26F6'VFTW6WFR66VEfFTFVƗfW'&WG'66VGVVBf6SFV'Vr%V&RF66VGVR66VB'&F67B7FFR&WG'"RvWDW76vRРР&fFR76&旦VBfB&WG'66VEfFTFVƗfW'W'67FV6Rf"7G&r6W'fW"vWEfFT66TFW"vWD66VEfFW56W'fW'2f"ffƖT'VvVUfFRfFRWr'&Ɨ7CvWEfFT66TFW"vWEfFW26W'fW"bfFR4FVƗfW'7FFTF'GW'67E6W'fW%fFTFVƗfW'6W'fW"fFRРРРf"7G&rWVBWrƖVD66WCvWEfFT66TFW"vWDƖUfFUUTG2f"ffƖT'VvVUfFRfFRWr'&Ɨ7CvWEfFT66TFW"vWDƖUfFW2WVBbfFR4FVƗfW'7FFTF'GW'67DƖUfFTFVƗfW'WVBfFRРРРРV&Ɩ2fB6V6fFU'GbvWD6frvWEfFU'GV&VBbfFU'GfFW27W'&VEfFU'GfFW5&WV&VBFV'Vr%fFR'G&V6VB"FD7W'&VEfFU'GfFW27W'&VEfFU'GfFW5&WV&VB7W'&VEfFU'GfFW5&WV&VBvWD6frvWEfFU'G7&V6UfFW5&WV&VB6WEfFT66UfFU'G7&V6UfFW5&WV&VBvWEfFT66UfFU'G7&V6UfFW5&WV&VBvWD6frvWEfFU'G7&V6UfFW5&WV&VBbvWD6frvWEfFU'G'&F67B4VG'&F67BvWD6frvWEfFU'G'&F67BРf"7G&r6BvWD6frvWEfFU'G'VvVT6G2'V66T6B6BРbvWD6frvWEfFU'G6VEF6W'fW'2f"7G&r6W'fW"vWDf&U6W'fW'26VEfFU'G6W'fW"РV6Rf"7G&r6W'fW"vWD6frvWEfFU'G6W'fW'5F6VB6VEfFU'G6W'fW"РРР6fUfFT66TfRРРV&Ɩ2'7G&7BfBFV'Vr7G&r7G"&fFRfBFV'Vs"7G&rW76vRFV'VrW76vRР򢠠EE6ƖVBW6VBf"r&WVW7G2ࠒ&fFRfGG6ƖVBGG6ƖVBGG6ƖVBWt'VFW"6V7EFVWBGW&Fe6V6G2R'VB򢠠fWF6W2W"w2UTBg&FRrࠒ&W$RW"P&WGW&W"UTB"6FRVbBfV@F&w2W6WFbFR&WVW7Bf0F&w2FW''WFVDW6WFbFW''WFVBvRvFrf"FR&W76PV&Ɩ2UTBfWF6UTB7G&rW$RF&w2W6WFFW''WFVDW6WFbW$RVW$RWV4v&T66R&V"&WGW&VðРGG&WVW7B&WVW7BGG&WVW7BWt'VFW"W&U$7&VFR&GG3r6W6W'2&fW2֖V7&gB"W$RtUBFVWBGW&Fe6V6G2R'VBGG&W76S7G&s&W76RGG6ƖVB6VB&WVW7BGG&W76R&GFW'2e7G&rb&W76R7FGW46FRC&W76R7FGW46FRCBr%FW&R2W"vFFRR""W$R%""&WGW&VðРb&W76R7FGW46FR#&W76R7FGW46FR3F&rWrW6WF$fVBFfWF6UTBf""W$R"EE"&W76R7FGW46FRР6VVVBVVVB6'6W"'6U7G&r&W76R&GbVVVBVVVVB46&V7B&WGW&VðР6&V7B&V7BVVVBvWD46&V7Bb&V7B2&B"&V7BvWB&B"46V‚&WGW&VðР7G&rWVD57G&r&V7BvWB&B"vWD57G&r&WGW&'6UUTDg&7G&rWVD57G&rРV&Ɩ2'7G&7B6WC7G&svWDf&U6W'fW'26WFRFf&6W'fW"6WB&Vf&RvFVƗ7B&6VB&WFrfFW'2V&Ɩ2'7G&7B6WC7G&svWD6fwW&VE6W'fW'2V&Ɩ2'7G&7BfFuVv&6frvWD6frV&Ɩ2'7G&7B7G&rvWD7W'&VEW%6W'fW"7G&rW"򢠠&W6fW2W"w26W'fW"f"fFR&WFrFVF6FVBfFr&26W'26BW6W2FR&6VB&W6V6RG&6W"7FVBࠒ&FV7FVB7G&rvWD7W'&VEW%6W'fW$f%fFU&WFr7G&rW"b4FVF6FVEfFu&V&VB&WGW&&6VEW%&W6V6UG&6W"vWEW"W"&W6V6R&W6V6RvWE6W'fW"$V6RV“Р&WGW&vWD7W'&VEW%6W'fW"W"Р򢠠FVF6FVB&WFr2FVFǒVf&RVvW76vsF@G&7'B2GF6VBFW"f6r&BFW2B6''&6V@&W6V6R66G2ࠒ&FV7FVB&V4FVF6FVEfFu&V&VB&WGW&vWD6frvWDFVF6FVEfFu&bbWFBVbbWFB7W'G4&6VE&W6V6RРV&Ɩ2'7G&7BfRvWDFFfFW%VvₓV&Ɩ27G&rvWDFFF5vFFFUF6FFUFR5FRvWD'VvVUFT6V6W"vWEFR&WGW&vWDFFF5vFFFUF5FRРV&Ɩ27G&rvWDFFF5vFFFUF6FFUFR5FR&WGW&$FFF"5FRvWDFF7G&r""5FRvWEV"РV&Ɩ2'7G&7B7G&rvWE&W$R7G&rWVB7G&rW$RV&Ɩ2'7G&7B7G&rvWEUTB7G&rW$R&fFRBvWEfVR'&Ɨ7C6V627G&r6VBFFBf"6VB62bBvWDRWV4v&T66R6V⒒FFfVRfVRBvWEfVRBVbfVRV’&WGW&FFCРbfVR4BVfVRvWDBV6RbfVR57G&rG'VFVvW"'6TBfVRvWE7G&r6F6W6WFRv&PРР&WGW&VFFCРР&WGW&FFCР&fFRfFUFF566BvWE&V7FVE&fW%FF2'&Ɨ7C6VFF7G&rW"Ɨ7CFUGSFT6vW2vWDv&FFFW"vWEFT6vW2&V&W6WDFFT6vW26F2FUGRD&V&W6WEvVVFT6vW26F2FUGRtTT&V&W6WDFFT6vW26F2FUGRDB66WFVEVWVVEfFW2B66WFVDv&VWVVEfFW2f"fFUFUVWVRVWVVBvWEfFT66TFW"vWEFT6vUVWVRbVWVVB5&6W76VB66WFVDv&VWVVEfFW2РbVWVVB5&6W76VBbbVWVVBvWDRVbbVWVVBvWDRWV4v&T66RW"66WFVEVWVVEfFW2РРBfFT7&VVB66WFVEVWVVEfFW2BFUFFvWEfVRFF$FUFF"fFT7&VVBBFFF&W6WDFfFT7&VVBvWEfVRFF$FFF"fFT7&VVBBvVVǕFF&W6WEvVVfFT7&VVBvWEfVRFF%vVVǕFF"fFT7&VVBBFǕFF&W6WDFfFT7&VVBvWEfVRFF$FǕFF"fFT7&VVBBG2vWEfVRFF%G2"fFT7&VVBvWD6frvWEG4fFRBfFW2vWD6frvWDVDefFW5W$FbfFW2BF2vWD'VvVUFT6V6W"vWEFRvWDFdFbFFFF2fFW2FFFF2fFW3РРbvWD6frvWDƖ֗EfFUG2bbG2vWD6frvWDƖ֗EfFUG2G2vWD6frvWDƖ֗EfFUG2РBFFTFFFbvWD6frvWE7F&TFFF5vFFFRbvWD6frvWEW6TFFFUFF45&'FF‚FFTFFF&W6WDFfFT7&VV@vWEfVRFFvWDFFF5vFFFUFfFT7&VVBV6RFFTFFFFFFðРРE&V7FVEfFU'GvWE&V7FVEfFU'G7FFR66WFVDv&VWVVEfFW2&WGW&WrfFUFF566BFUFFFFFvVVǕFFFǕFFG2&V7FVEfFU'G&V7FVEfFU'GFFTFFF“Р&FV7FVB&V6f'v&E7FFT'&F67B&VvW5FF2&WGW&vW5FF3Р&FV7FVBEvWE&V7FVEfFU'G7FFRB66WFVEfFW2B7W'&VBfFU'GfFW3B&WV&VB7W'&VEfFU'GfFW5&WV&VCbvWD6frvWEfFU'GV&VB&WGW&WrE7W'&VB&WV&VBӰРB7&V6RvWD6frvWEfFU'G7&V6UfFW5&WV&VBf"B66WFVEfFW37W'&VBb7W'&VB&WV&VB7W'&VB&WV&VC&WV&VB7&V6SРР&WGW&WrE7W'&VB&WV&VBӰРV&Ɩ2'7G&7B7G&rvWEVvfW'6ₓV&Ɩ2'7G&7BBvWEfFT66T7W'&VEfFU'GfFW2V&Ɩ2'7G&7BrvWEfFT66T7EWFFVBV&Ɩ2'7G&7BBvWEfFT66U&WdFV&Ɩ2'7G&7B7G&rvWEfFT66U&WdFV&Ɩ2'7G&7BBvWEfFT66U&WevVVV&Ɩ2'7G&7BBvWEfFT66UfFU'G7&V6UfFW5&WV&VBV&Ɩ2'7G&7B&V5W$ƖR7G&rW$R򢠠6V62ƖR7FFRf"fFR&WFrW6r&6VB&W6V6RǒvVF0&2WƖ6Fǒ6fwW&VB2FRFVF6FVBfFr&ࠒ&FV7FVB&V5W$ƖTf%fFU&WFr7G&rW$R&WGW&4FVF6FVEfFu&V&VB&6VEW%&W6V6UG&6W"vWEW"W$R5&W6VB5W$ƖRW$RРV&Ɩ2'7G&7B&V56W'fW%fƖB7G&r6W'fW"V&Ɩ2'7G&7B&V56VTƖU6W'fW"7G&r6W'fW"&FV7FVB&V56VTƖU6W'fW$f%fFU&WFr7G&r6W'fW"b4FVF6FVEfFu&V&VB&WGW&56VTƖU6W'fW"6W'fW"Р6&V6FWfFwVv&&W6V6R&6VE&W6V6U7FGW27FGW2&6VEW%&W6V6UG&6W vWD&6VE7FGW26W'fW"&WGW&7FGW2Vbb7FGW24f&Rbb7FGW2vWEW$6VBРV&Ɩ2'7G&7B&V5fFT66Tv&UFRV&Ɩ2'7G&7Bח76frvWEfFT66Tו56frV&Ɩ2'7G&7Bח76frvWDfFVD66Tו56frV&Ɩ2'7G&7Bח76frvWEfFTvvtו56fr򢠠6WFFvו5&VFVB&W6W&6W26fVǒࠒV&Ɩ2fB6WFFvו7‚bvWE&ח7W76VvW"V’vWE&ח7W76VvW"6WFFvₓ6WE&ח7W76VvW"V“РbvWE&ו5‚V’vWE&ו5‚6WFFvₓ6WE&ו5†V“РРV&Ɩ2fBBfFT66R67F&vRfFVEW'57F&vRfFVD66T6WFB'VvVTWFBvWD'RvWD6frvWD'VvVTWFBbvWDWFBV’WFB'VvVTWFBTtU54tsРv&V7W'FVDFVF6FVEfFu&FRWVEW$T66RvWE&ו5‚vWE&w5UTDUVW''VvVUFT6V6W"6WEFT6vTf6fT'72vWD6frvWEFT6vTf6fT'72'VvVUFT6V6W"EFW"fFT66TFW"WrfFT66TFW"vWEfFT66Tו56frvWD6frvWEfFT66UW6Tו5‚vWD6frvWEfFT66UW6Tו5‚vWE&ו5‚vWDח7‚vWD6frvWDFV'Vr67F&vRfW'&FPV&Ɩ2fBtf7G&r6rtf6rРfW'&FPV&Ɩ2fBu6WfW&S7G&r6ru6WfW&R6rРfW'&FPV&Ɩ2fBFV'VsW6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vs7G&r6rbvWD6frvWDFV'VrFV'Vr6rРРfW'&FPV&Ɩ2fBFV'VsF&v&RRbvWD6frvWDFV'VrR&E7F6G&6RРРӰfFT66TFW"BfFVEW'466RWrfFVEW'466RvWDfFVD66Tו56frvWD6frvWDfFVD66UW6Tו5‚vWD6frvWDfFVD66UW6Tו5‚vWE&ו5‚vWDח7‚fFVD66T6vWD6frvWDFV'VrfW'&FPV&Ɩ2&VW6W$W7G27G&rWVB&WGW&vWE&ו5‚6F4WVW'WVBРfW'&FPV&Ɩ2fBtf7G&r6rtf6rРfW'&FPV&Ɩ2fBu6WfW&S7G&r6ru6WfW&R6rРfW'&FPV&Ɩ2fBFV'VsW6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vs7G&r6rbvWD6frvWDFV'VrFV'Vr6rРРfW'&FPV&Ɩ26WC7G&svWDUTG2&WGW&vWE&ו5‚vWEWVG2РӰbWFBWV2'VvVTWFBՕ5’G'&ח7W76VvW"Wrו7W76VvW"%fFuVv"vWE&ו5‚vWDח7‚vWD6V7FvW"vWDFF6W&6Rו7W76VvW"FR$V6W'fW$B$FP6rbvWD6frvWDFV'VrFV'Vr$vBg&"6r6W&6R#"6rVfVRvWE7V$6V‚" 6rVfVRvWDfVG2Рv&W76vU&FW"W76vR6rVfVRғ6F65W6WFRR&E7F6G&6RРV6RbWFBWV2'VvVTWFBTtU54trbvWD6frvWEVvW76vTV7'FₒV7'FFW"WrV7'FFW"%fFuVv"WrfRvWDFFfFW%Vvₒ'6V7&WFWW"РV6RbWFBWV2'VvVTWFB44UE2V7'FFW"WrV7'FFW"%fFuVv"WrfRvWDFFfFW%Vvₒ'6V7&WFWW"66WDFW"Wr66WDFW"vWEVvfW'6ₒvWD6frvWD'VvVT7BvWD6frvWD'VvVU'BV7'FFW"vWD6frvWDFV'VrfW'&FPV&Ɩ2fBr7G&r7G"tf7G"РӰ66WDFW"FBWr66WE&V6VfW"fW'&FPV&Ɩ2fB&V6VfTVfVR6VfVRVfVRv&W76vU&FW"W76vR6ۛh춻q^u͕!Yܤ($$%ɕɸ($%($%幍ɽ镐Aɕ͕!̤($$%չAAɕ͕!̡ܤ($$%Aɕ͕!̹СUե􁡅($$$%ɕՕ%ɕՕ%($$%($%(%((%ɥمєAɕ͕!YAAɕ͕!ܤ($%ɕɸձUեձѥ%ձ($$$$MՕ0($$$$܀􁡅ɕѕЀ܀ɕѕЀAIM9 }!9=}Q%5=UQ}5%11%L(%((%ɥمєMUU%ѕAAɕ͕!̡UU%ɕՕ%Mɥ͕ٕȰUU%%ɹѥ%($$%Mхѕаܤ($%1AAɕ͕!ѕ􁹕܁Ʌ1($%MUU%ѕÀ􁹕܁1!͡M($%幍ɽ镐Aɕ͕!̤($$%չAAɕ͕!̡ܤ($$%Aɕ͕!̹MРɕٕ%䀴($$$%AAɕ͕!􁕹九YՔ($$$%ɕՕ%Յ̡ɕՕ%($$$$%ɕɸ͔($$$%($$$%͕ٕȹՅ%ɕ ͔͕ٕȤ($$$$$$%ɹѥ%Յ̡%ɹѥ%($$$$$$MхѕЀ􁉅MхѕФ($$$$%ѕ($$$%($$$%ɕɸՔ($$%($%($%ȀAAɕ͕!聍ѕ($$%AAɕ͕ɕ͕􁉅AAɕ͕QɅȹAȡUե͔ձ($$%ɕ͕ձɕ͕MٕȠՅ%ɕ ͔͕ٕȤ($$$$$ɕ͕ ѥ%Յ̡ѥ%($$$%9ե͕ٕȤ($$$%ѕA̹Uե($$%($$%ɕ͕ѥѥ ($%($%ɕɸѕA(%(($($Ʌ́ٽѕȵ啐ɕ݅ɑ́ݡєɕٕ͹͡Ё($ɵ́ȁѕٽѥɽ丁 ɽ̵́ɔ($ɕɽ͕ѡȁѽչѥѠɔ፱ՑѼ($ٽ͕($(%ɽѕѕٽɽѕM͡1̡Mɥ͕ٕȰMUU%A̤($%ѕYѥAɽ͕ٕȀձ͕ٕȹ ($$%ɕɸ($%($%MUU%፱Ց􁡅Àձ ѥ̹MР聡A($%ȀAAɕ͕ɕ͕聉AAɕ͕QɅȹ=A̠($$%ɕ͕MٕȠՅ%ɕ ͔͕ٕȤ፱Ցх̡ɕ͕Uե($$$%ɕ͕A9ɕ͕UեѽMɥɕ͕MٕȠ($$%($%(%((%ɥمєٽ͍ɑAAɕ͕!Mɥե($%($$%UU%UեUU%ɽMɥեɥ($$%AAɕ͕!ɕٕ($$%幍ɽ镐Aɕ͕!̤($$$%ɕٕAɕ͕!̹ɕٔUե($$%($$%ɕ͕ѥѥ ɕٕ($%􁍅эፕѥɕ($$$%مѥѥ́ɔɕѕѡɕ͕Ʌȸ($%(%((%ɥمєٽ͍ɑAAɕ͕!AAɕ͕!($%ɕٕ􁙅͔($%幍ɽ镐Aɕ͕!̤($$%ձAɕ͕!̹СUե􁡅($$$%Aɕ͕!̹ɕٔUե($$$%ɕٕՔ($$%($%($%ɕٕ($$%ɕ͕ѥѥ ($%(%((%ɥمєٽ͍ɑAAɕ͕!̡Mɥ͕ٕȤ($%幍ɽ镐Aɕ͕!̤($$%Aɕ͕!̹MРɕٕ%䀴($$$%九YՔ͕ٕȹՅ%ɕ ͔͕ٕȤ($$$$%ɕɸ͔($$$%($$$%ɕ͕ѥѥ 九YՔ($$$%ɕɸՔ($$%($%(%((%ɥمєٽ͍ɑAAɕ͕!̡UU%ɕՕ%($%ɕՕ%ձ($$%ɕɸ($%($%幍ɽ镐Aɕ͕!̤($$%Aɕ͕!̹MРɕٕ%䀴($$$%ɕՕ%Յ̡九YՔɕՕ%($$$$%ɕɸ͔($$$%($$$%ɕ͕ѥѥ 九YՔ($$$%ɕɸՔ($$%($%(%((%ɥمєٽչAAɕ͕!̡ܤ($%Aɕ͕!̹MРɕٕ%䀴($$%AAɕ͕!􁕹九YՔ($$%܀􁡅ɕѕЀ܀ɕѕЀAIM9 }!9=}Q%5=UQ}5%11%L($$$%ɕɸ͔($$%($$%ɕ͕ѥѥ ($$%ɕɸՔ($%(%((%ɥمєٽɕ͕ѥѥ AAɕ͕!($%ձ($$%AAɕ͕QɅȹɕ͕ѥѥ Uե͕ٕȰ($$$$%MՕ($%(%((%ɽѕѕٽɕAAɕ͕!̡ܤ($%1AAɕ͕!ɕ􁹕܁Ʌ1($%幍ɽ镐Aɕ͕!̤($$%չAAɕ͕!̡ܤ($$%ȀAAɕ͕!Aɕ͕!̹مՕ̠($$$%UU%ѥٕIՕ%􁉅AAɕ͕QɅȹAM͡IՕ%͕ٕȰܤ($$$%ɕՕ%ձɕՕ%Յ̡ѥٕIՕ%($$$$%ɕՕ%ձ($$$%($$$%ɕՕ%ձ($$$$%ɕ久($$$%($$%($%($%ȀAAɕ͕!ɕ䤁($$%ɕՕ Aɕ͕M͡С͕ٕȰܰՔ($%(%((%ɽѕѕЁAAɕ͕! չР($%幍ɽ镐Aɕ͕!̤($$%ɕɸAɕ͕!̹ͥ锠($%(%((%ɽѕѕٽ͍ձ Aɕ͕MхI幌($%MձᕍѽM٥͍ձȀ􁝕MձȠ($%ѡձѡ Aɕ͕͍ձȀձ($$%ɕɸ($%($%͍ձȹ͍ձѡɕՕ Aɕ͕MхI幌($$$%AIM9 }MQIQUA}IMe9 }1e}M =9LQUйM =9L(%((%ɽѕѕٽɕՕ Aɕ͕MхI幌($%ѡձѡ Aɕ͕5ͅAɽ!Ȁձ($$%ɕɸ($%($%ɕՕѕЀMѕɕQ5̠($%Ё($%ȀMɥ͕ٕȀ聝مMٕ̠($$%Aɕ͕MٕY͕ٕȰYѥA՝]ɔMU }AIM9 }IMe9 }IEUMP($$$%ѥՔ($$%($$%5ͅAɽ!ȹ͕5͕ٕͅȰ䬬($$$$%YѥA՝]ɔɕ͕I幍IՕС͕ٕȰUU%ɅUU%ɕՕѕФ($%(%((%ɥمєٽх Aɕ͕($%ѡձѡ Aɕ͕($$%ɕɸ($%($%ɕ Aɕ͕AIM9 }  -9}Q%5=UQ}5%11%L($%ȀMɥ͕ٕȀ聝مMٕ̠($$% IٕM̹͡х̡ɕ͕Mٕ-͕ٕȤ($$$%ɕՕ Aɕ͕M͡С͕ٕȤ($$%($%($%ɕAAɕ͕!̡MѕɕQ5̠(%((%ɥمєMɥɕ͕Mٕ-Mɥ͕ٕȤ($%ɕɸ͕ٕȀձ͕ٕȹɥѽ1ݕ ͔لѥ1I==P(%((%ɥمєAɕ͕MٕYMɥ͕ٕȰMɥՉ ($$Qɕ͕ɽѽ́Ёչ䁥́ѡɕ͕иQ($$͕ѕɅЁЁ䁉ͥѼ͕ٕ́ѕЁѼ($$ͽєѡȸ($%͕ٕȀձ͕ٕȹ MٕY͕ٕȤ($$%՜%ɕՉ ɕ͕ٕȁչɕ͕ٕȈ($$%ɕɸ͔($%($%ɕɸՔ(%((%ɥمєAɕ͕ɅѥYUU%%ɹѥ%Mхѕа($$%ɕ͕QхMɥՉ ($%%ɹѥ%ձMхѕЀ0ɕ͕QхMхѕФ($$%՜%ɕՉ ɕ͕ٕݥѠمɅѥ($$%ɕɸ͔($%($%ɕɸՔ(%(($($Íٕɕ͕ݹ䁉́ѡЁٔѽɕѥщ̸($MձѥЁɅѥɔѕѥ䁱ЁѼѕ($ɽ䁵($($Ʌѥ5́᥵մͥɔ($ɕɸɕ͕ٕȁ($(%ՉMMɥɕ Aɕ͕ѥ5̤($%ѡձѡ Aɕ͕($$%ɕɸ ѥ̹MР($%($%܀MѕɕQ5̠($%MMɥɕ􁉅AAɕ͕QɅȹɕ ̡ܰѥ5̤($%ȀMɥ͕ٕȀ聕ɕ($$%͍ɑAAɕ͕!̡͕ٕȤ($$$-ɕٕݡѡ́Ʌѥ́չم%ѡͅ($$$ɽ́ɕյ̰́щЁɬЁمѡ($$$ѕхͬݥɕՕЁɕ͠͹͡Ё́ݡх啐($$% IٕM̹͡ɕ͕Mٕ-͕ٕȤ($%($%幍ɽ镐Aɕ͕!̤($$%չAAɕ͕!̡ܤ($%($%ɕɸɕ(%((%ՉٽMɥ9MɥեMɥ͕ٕ9($% =5($$%ե􁝕UU%9($%(($%($$%եձե䠤եՅ%ɕ ͔ձ($$$%եUU%ɽMɥեɥѽMɥ($$%($%􁍅эፕѥɕ($$$ɔ($%(($% =5($$%9YѕAȡե9($%($%A=YѕIѥ9($$% х($$$%х!ȠQ !($$$$%х!Ƞ͡Q ̠($$$%($$%(($$% Yѕ̡͕ٕ9($$%ɕA= ɽ̡͕ٕ9($$%ɕAQ ɽ̡͕ٕ9($$%=Yѕ̡9ե͕ٕ9($$%ձѥAɽ!ȹե9($%(%((%ɥمєٽ%Mɥ͜($%͜(%((%ՉɅЁٽMٕɔMɥͅ((%Չٽͅ($%ѽ ɽM٥̡͔($%Yѕ !ȠٕͅYѕ (($%Aɽ5ű5͕Ƞձ($$%Aɽ5ű5͕Ƞ͡ёݸ($%(($%Aɽ5ME0ձ($$%Aɽ5ME0͡ёݸ($%($%ձѥAɽ!Ȁձ($$%ձѥAɽ!ȹ͔($%(($%ͽ!Ȁձ($$%ͽ!ȹ͕ ѥ($%(($%ɕ!Ȁձ($$%ɕ!ȹ͔($%($%ɕAՉ͡Aձ($$%ɕAՉ͡A͔($$%ɕAՉ͡Aձ($%(($%չQ ȹ͡ёݸ(($%х!Ƞձ($$%х!Ƞ͡ёݸ($%(($%􁙅͔(%((%ՉٽA՝5ͅIٕх%Mɕ($%չ幌($$%($$$%MɥMՈ($$$% A՝5ͅѥѥ!Ȁձ($$$$%MՈ􁕹ѥ!ȹСɕUQ($$$%􁕱͔($$$$%MՈ􁥸ɕUQ($$$%(($$$%Ёͥ销􁥸ɕ%Р쀼ͅ䁽(($$$% ՜($$$$%՜Iٕ՝ͅ􈀬MՈͥ􈀬ͥ锤($$$%(($$$%Mɥ屽􀈈($$$%ͥ销($$$$% A՝5ͅѥѥ!Ȁձ($$$$$%屽􁕹ѥ!ȹСɕUQ($$$$%􁕱͔($$$$$%屽􁥸ɕUQ($$$$%($$$%(($$$%)ͽٕٕ)ͽٕ 屽(($$$%MՈՅ%ɕ ͔ٕMՉ ($$$$% ՜($$$$$%݅ɸA՝5ͅՉ ͵э聡􈀬MՈ($$$$$$$$ٕMՉ ($$$$%($$$$%ɕɸ($$$%(($$$%5ͅAɽ!ȹ5ٕͅ($$%􁍅эፕѥ($$$%ɥMхQɅ($$%($%(%((%ɥمєUU%͕UU%ɽMɥMɥեMɥ($%Mɥmt̀쀈եMɥՉɥरեMɥՉɥఀȤ($$$$եMɥՉɥȰؤեMɥՉɥذ($$$$եMɥՉɥȤ(($%M ̀1ltYՔ($%M ̀($%M ́1ltYՔ($%M ̀($%M ́1ltYՔ(($%M ̀1ltYՔ($%M ̀($%M ́1ltYՔ(($%ɕɸ܁UU%M ̰M ̤(%((%Չ幍ɽ镐ٽɽEՕՔ($%ݡYѕ !ȠQ EՕՔͥ锠($$%YѕQEՕՔٽє􁝕Yѕ !ȠQ EՕՔР($$%ٽєAɽ͕($$$%YѕQхM͡ЁՕՕQх̀ٽєQх̠ձٽєQх̠䠤ձ($$$$$$YѕQхM͡й͕MѽɅٽєQх̠($$$%EՕՕYѕIձЁɕձЀٽєٽє9ٽєM٥Ք͔ٽєQՕՕQх̰($$$$$%ٽєUեٽє($$$%ɕձЀEՕՕYѕIձйIQId($$$$%͍ձQYѕI䠤($$$$%ɕɸ($$$%($$$%ɕձЀEՕՕYѕIձйQI5%90($$$$%݅ɸI٥ѕɵɽٕȁٽєٽєYѕ%Ȁٽє9($$$$$$$M٥MѕYѽȹͅѥ1ٽєM٥($$$%($$%($$%Yѕ !ȠɕٕQYєٽє($$$%͍ձQYѕI䠤($$$%ɕɸ($$%($%(%((%ɥمєٽ͍ձQYѕI䠤($%ѥYѕIMձMձȠձ($$%ɕɸ($%($%ѥYѕIMձՔ($%($$%MձȠ͍ձ($$$%幍ɽ镐YѥA՝Aɽѡ̤($$$$%ѥYѕIMձ􁙅͔($$$%($$$%ɽEՕՔ($$%԰QUйM =9L($%􁍅эIչѥፕѥ($$%ѥYѕIMձ􁙅͔($$%՜UѼ͍ձɽٕȁٽєɕ耈5ͅ($%(%((%Չٽɕ($%ɕIչѥՔ(%(($́ ɽɥѕɅѥɕݥѡЁѽ́ѽȁȁѕ͕٥(%Չٽɕɽ ɽ($%ɕIչѥ͔(%((%ɥمєٽɕIչѥɕх ɽM٥̤($%ѡ չ5ѡ 9 չ5ѡ($%5ѡձ($$%ѡ չ5ѡA1U%95MM%9($%($%݅ɹUѕѕYѥAɽ5(($%͕ ɕYѕAYѕIեɕ($$$% YѕAYѕIեɕYѕ YѕA%ɕ͕YѕIեɕ($%5ձѥAɽMР($%ɕх ɽM٥̤($$%х ɽM٥̠($%(%((%ɥمєٽ݅ɹUѕѕYѥAɽ5($% ѕYѥAɽ䠤ѡձѡ Aɕ͕($$%MٕɔѕYѥAɽɕեɕ́5eME0I%L5EQPȁM= -QLA1U%95MM%9́ͅȀ($$$$$ѕɽɽѥѼɵɽɽѥ($%(%((%ՉɅЁٽչ幌Iչո(($Aљɴ͕䁙ȁѡɅеɅ ɽ͍ٕ䁍Ʌи(%ՉɅЁMɥAɽAљɴ((%ՉɅЁٽչ ͽ Mɥ((%ՉɅЁٽٕͅYѕ ((%ՉɅЁٽɕ ɔŰ(($MɥЁ ɽɕѠ쁙ɕ́ɽєͼѡȁɕѽɔ́(%ՉɅЁٽɕ ɽ Ʌѥѡɽ́ፕѥ((%ՉɅЁ͕A՝5ͅфMɥ͕ٕȰMɥѕmtфՕՔ((%ɥمєхѥЁA1U%9}5MM}!I}1%5%P(%ɥمєхѥЁA1U%9}5MM}M=Q}1%5%P((%Չٽ͕A՝5ͅMٕȡMɥ͕ٕȰЁ䰁)ͽٕٕ($%MձȠ͍ձ͕A՝5ͅMٕ9ܡ͕ٕȰٕ䀨0QUй5%11%M =9L(%(($($Ḿ՝ٕͅѕ䁅ɕ́ݡѡȁѡɽ($ѕЁȁٕ($($Ʌ͕ٕȁхɝЁ͕ٕ($ɅٕٕѼ͕($ɕɸՔݡѡɽ䁅ѕѡͅȁٕ($(%ɽѕѕ͕A՝5ͅMٕ9ܡMɥ͕ٕȰ)ͽٕٕ($%MɥՉ 􁕹ٕMՉ ($%Mɥ屽)ͽٕ ٕ(($%ѕmtՉ ѕ̀Չ ѕ̡ل͕йMхɑ ͕̹UQ|($%ѕmt屽 ѕ̀屽 ѕ̡ل͕йMхɑ ͕̹UQ|(($$ѥєѕ́ɥѕ($$ɥѕUQ̀ȵєѠɕUQѕ($$ɥѕ%Ё̀Ёѕ($%ЁѥѕM销ȀՉ ѕ̹ѠՉ UQɕѕ̤($$$$Ѐ屽Ѡ($$$$Ȁ屽 ѕ̹Ѡ쀼屽UQɕѕ̤(($%ѥѕM销A1U%9}5MM}M=Q}1%5%P($$%՜mA՝5ͅtA屽ɥЀѥѕM销ѕ̤͕ٕ􈀬͕ٕ($$$$$Չ 􈀬Չ PͥȁÍѕ($%(($%ѥѕM销A1U%9}5MM}!I}1%5%P($$%՜mA՝5ͅtA屽Q=<1IѥѕM销ѕ̰􈀬A1U%9}5MM}!I}1%5%P($$$$$͕ٕ􈀬͕ٕȀՉ 􈀬Չ P9=P͕Ј($$%ɕɸ͔($%(($%䀡 ѕɅ=Mɕѕ=Mɕ􁹕܁ ѕɅ=Mɕ($$$%х=MɕЀ􁹕܁х=Mɕѕ=Mɕ($$% A՝5ͅѥѥ!Ȁձ($$$%йɥѕUQѥ!ȹСՉ ($$%􁕱͔($$$%йɥѕUQՉ ($$%(($$$ͅ䁽5UMPѕ̰Ё($$%йɥѕ%С屽 ѕ̹Ѡ(($$% A՝5ͅѥѥ!Ȁձ($$$%йɥѕUQѥ!ȹС屽($$%􁕱͔($$$%йɥѕUQ屽($$%($$%й͠(($$%͕Ѐ͕A՝5ͅф͕ٕȰ A՝5ͅ ѽ1ݕ ͔($$$$%ѕ=Mɕѽ ѕɅ䠤͔($$% ՜($$$%՜͕ЀMЈ耉 ձЁ͕՝ٕѥѕM销ѕ̤͕ٕ($$$$$$Չ ٕ̠($$%($$%ɕɸ͕($%􁍅эፕѥ($$%ɥMхQɅ($$%ɕɸ͔($%(%((%ɥمєձ) եI ($%ձ) եȁձ) եȠ($$$$х͔ I%ࠤѥQ5̠ͽQ5̠($% IU͕ɹձ IU͕ɹ䠤($$%͕ȡ IU͕ɹ($%($% IAݽɐձ IAݽɐ䠤($$%ݽɐ IAݽɐ($%($%ɕɸե(%((%Չ͕IٕMٕȡMɥ͕ٕȰ)ͽٕٕ($%ɕɸ͕IٕMٕȡ͕ٕȰٕ͔(%((%ɥمє͕IٕMٕȡMɥ͕ٕȰ)ͽٕٕ͕I ݸ($%)AՉ͡AɕAՉ͡A($%Չ͡Aձ͕I ݸMѕɕQ5̠ɕAՉ͡IѕȤ($$%ɕɸ͔($%(($%䀡)́̀Չ͡AIͽɍ($$%Mɥ􁝕 IAɕࠤYѥA՝|͕ٕ($$%Չ͍ɥ̀􁩕̹Չ͠)ͽٕ ٕ($$%ɕAՉ͡IѕȀ0($$%ɕɸՉ͍ɥ̀($%􁍅эፕѥ($$%͕I ݸ($$$$Mхɽ́ɕՕՕͼѡȁɕɥ́ѡɽѱͅ($$$%ɕAՉ͡IѕȀMѕɕQ5̠0($$%($$%՜5ͅ($$%ɕɸ͔($%(%((%Չ͕5ٕMٕȡMɥ͕ٕȰ)ͽٕٕ($%!Ȁձ($$%ɕɸ͔($%($%($$%!ȹՉٕ͡ 5Aɕࠤٽѥ՝͕ٕ̼͕ٕȰٕ($$%ɕɸՔ($%􁍅эፕѥ($$% ՜($$$%ɥMхQɅ($$%($$%ɕɸ͔($%(%((%Չ͕MٕMٕȡMɥ͕ٕȰ)ͽٕٕ($%5Mɥ=Ʌѥ􁝕 MMٕ Ʌѥ͕ٕȤ($%Ʌѥձ($$%ɕɸ͔($%($%MɥЀ􁍽ɅѥР!ЈхMɥMɥɅѥР!Ј耈($%ЁЀ􁍽ɅѥРAЈх9յȀ9յȤɅѥРAЈYՔ($%й䠤($$%ɕɸ͔($%(($%Mɥ屽)ͽٕ ٕ($%Mɥ􁕹ѥ!Ȁձѥ!ȹС屽屽($%䀡MЁͽЀ􁹕܁MР($$%ͽйС܁%Mɕ̡аФ($$%䀡х=MɕЀ􁹕܁х=Mɕͽй=Mɕ($$$%йɥѕUQ($$$%й͠($$%($$%ɕɸՔ($%􁍅эፕѥ($$%՜5ͅ($$%ɕɸ͔($%(%((%Չٽ͕Mٕ95ͅ($%ȀMɥ̀聝مMٕ̠($$%͕A՝5ͅMٕȡ̰İYѥA՝]ɔ͕ٕ9̤($%(%((%Չٽ͕YѕAMɥ͕ٕȤ($%M=MٕYѕIѥ͕ٕȤ($$%5ͅAɽ!ȹ͕5͕ٕͅȰİYѥA՝]ɔٽѕA չ($%(%((%Չٽ͕ ɕYѕAYѕ̡ЁչФ($%ٽѕAYѕ̀􁅵չ($%͕Yѕ YѕA ɕYѕ̡չФ($%՜ ɕЁٽєѽх耈ٽѕAYѕ̤(%((%ՉɅЁٽ͕Yѕ 1Uѕ((%ՉɅЁٽ͕Yѕ Aɕ䡥Ё((%ՉɅЁٽ͕Yѕ Aɕ5ѠMɥѕФ((%ՉɅЁٽ͕Yѕ Aɕ]Ёݕ((%ՉɅЁٽ͕Yѕ Yѕ %ɕQɔ((%ՉɅЁٽ͕Yѕ YѕA ɕYѕ̡Ёٽѕ̤((%ՉɅЁٽ͕Yѕ YѕA%ɕ͕YѕIեɕЁٽѕ̤((%Չٽх̠($%ȀMɥ̀聝مMٕ̠($$%M=MٕYѕIѥ̤($$$%9͕ٕ́Ȁ̀Ѽ͕ѕЁх͔́ͅɕѕЁݥѠͽ($$%􁕱͔($$$%MɕՕЁȁх̤́ͅ($$$%5ͅAɽ!ȹ͕5̰ͅİYѥA՝]ɔх̡̤($$%($%(%((%ɥمєٽ͕YѕIѕMɥȰMɥեMɥ͕٥=($$%MɥMٕȤ($%=MٕȀձمMٕ̠х̡MٕȤ($$%՜9Ё͕ٽєɕѥȀȀ͔ѡȁ́($$%ɕɸ($%(($%5ͅAɽ!ȹ͕5ͅMٕȰİ($$$%YѥA՝]ɔٽѕIѕȰե͕٥Ք(%((%ՉMɥ]UѥMѕɽM٥Mɥ͕٥($%ȀMɥͥє聝 ]UѥYѕMѕ̠($$% ]UѥYѕM٥ͥєՅ%ɕ ͔͕٥($$$%ɕɸͥє($$%($%($%ɕɸ(%((%ɥمє1YѕQMɥեɅ1 յ̰MɥͥєMɥ͕٥MɥȰ($$%ՑQ EՕՔ($%IQ(($%Yѕ !Ƞ=Yѕ̡ե($$%Ʌ1= չYєYѕ̀􁝕Yѕ !Ƞ=Yѕ̡ե($$%Ȁ= չYєٽє聽Yѕ̤($$$%ٽєM٥Յ%ɕ ͔͕٥($$$$%IQ5ѠࡵIQٽєQ($$$%($$%($%(($%ȀMɥ͕ٕȀ聝مMٕ̠($$%Ȁ= չYєٽє聝Yѕ !ȠYѕ̡͕ٕȤ($$$%ٽєUեՅ̡եٽєM٥Յ%ɕ ͔͕٥($$$$%IQ5ѠࡵIQٽєQ($$$%($$%($%(($%ՑQ EՕՔȀձ($$%ȀYѕQEՕՔՕՕYє聝Yѕ !ȠQ EՕՔ($$$%ՕՕYє9Յ%ɕ ͔Ȥ($$$$$$ՕՕYєM٥Յ%ɕ ͔͕٥($$$$%IQ5ѠࡵIQՕՕYєQ($$$%($$%($%(($%Ȁ յ聍̤($$%9Յ%ɕ ͔1Yѕ̈($$$%хYՔمՔ􁐹YՔ($$$%MɥmtЀمՔMɥР($$$%ȀMɥȀ聱Ф($$$$%MɥmtфȹР($$$$%хltՅ%ɕ ͔ͥє($$$$$%IQ5ѠࡵIQ1مՕ=хlt($$$$%($$$%($$%($%($%ɕɸIQ(%((%ՉYѕMɥեMɥ͕٥Ʌ1 յф($%ɕɸYѕեձ͕٥ф͔(%(($($ ́ѡɕٽє䰁ѥ䁥Ցѕٽѕ́݅ѥ($ȁфѥѼ͠($($ɅեȁUU%($Ʌȁȁ͕ѡѥՕՔ($Ʌ͕٥ٽє͕٥($ɅфɕЁȁф($ɅՑQ EՕՔݡѡȁՕՕٽѕ́ɕ͕ٔѡȁͱ($ɕɸՔݡѡٽє䁉ѕ($(%ՉYѕMɥեMɥȰMɥ͕٥Ʌ1 յф($$%ՑQ EՕՔ($%Mɥͥє􁝕]UѥMѕɽM٥͕٥($%ͥє䠤($$%՜9͕٥ͥє͕ЁȀ͕٥ͭٽє䁍($$%ɕɸՔ($%(($%Ёٽѕ􁝕 ]UѥYѕYѕͥє($%Ёٽѕ5􁝕 ]UѥYѕYѕ5ͥє(($%Yє􁝕1YѕQեфͥє͕٥ȰՑQ EՕՔ($%Yє($$%՜9ЁٽєѥչȀե͕٥ͭٽє䁍($$%ɕɸՔ($%(($%($$%1ѕQ܀􁝕 չQ ȠQ($$%1ѕQYѕQ1ѕQ%хС%хй5Yєi%ѕձР($$$$$!̡ Q!=MР(($$% ]UѥYѕYѕͥє($$$%ٽѕٽѕ5($$$$%՜Yє䁥̀Ȁͥєͭٽє䁍($$$$%ɕɸՔ($$$%(($$$%1ѕQٽє􁱅YѕQ!̠ٽѕ䤹5ѕ̠ٽѕ5($$$%ɕɸܹѕȡٽє($$%($$%1ѕQɕ͕Q􁱅YѕQݥѡ!ȡ ]UѥYѕYѕ!ȡͥє($$$$$ݥѡ5єݥѡM($$%1ѕQɕ͕QQɽ܀ɕ͕Q!̠Ф(($$%YѕQ ɔɕ͕Q($$$%ܹѕȡɕ͕Q($$$$%՜Yє䁥́ЁȀե͕٥ٽєɽ͕($$$$%ɕɸՔ($$$%($$%􁕱͔($$$%ܹѕȡɕ͕QQɽܤ($$$$%՜Yє䁥́ЁȀե͕٥ٽєɽ͕($$$$%ɕɸՔ($$$%($$%($%􁍅эፕѥ($$%ɥMхQɅ($%(($%՜Yє䁥́ЁЁȀե͕٥ͭٽє($%ɕɸ͔(%((%Չ幍ɽ镐ٽٽєMɥȰMɥ͕٥ɕYєѥEՕՔՕՕQ($$%YѕQхM͡ЁѕаMɥե($%ٽєȰ͕٥ɕYєѥEՕՔՕՕQѕаեձ(%((%ɥمєմEՕՕYѕIձЁ($%MU MLIQIdQI5%90(%((%ɥمє幍ɽ镐EՕՕYѕIձЁٽєMɥȰMɥ͕٥ɕYєѥEՕՔՕՕQ($$%YѕQхM͡ЁѕаMɥեYѕQEՕՔՕՕYє($%($$%M٥MѕYѽȹY͕٥($$$%݅ɸIѕٽєݥѠم͕٥ͥєM٥MѕYѽȹͅѥ1͕٥($$$%ɕɸEՕՕYѕIձйQI5%90($$%($$%5ɅU͕ɹYѽȹYȰ ɽAAɕࠤ($$$%݅ɸIѕٽєݥѠم5ɅЁ͕ɹ($$$$$$5ɅU͕ɹYѽȹͅѥ1Ȥɽ͕٥($$$$$$5ɅU͕ɹYѽȹͅѥ1͕٥($$$%ɕɸEՕՕYѕIձйQI5%90($$%(($$%UU%ٽѕ%ՕՕYєձձՕՕYєYѕ%($$%ٽѕ%ձ($$$%ٽѕ%UU%ɅUU%($$%(($$$UU%ɕͽѥ($$% =5($$$%ե􁝕UU%Ȥ($$%(($$%եձե䠤($$$%ե􁝕UU%Ȥ(($$$$ ɽɕѼѕ($$$%ե䠤 ɽAAɕࠤ䠤($$$$$$ȹх]Ѡ ɽAAɕࠤ($$$$%MɥեĀ􁝕UU% ɽAAɕࠤȤ($$$$%եĹ䠤($$$$$%՜ѕѕɽȁݥѡЁɕఁѥ($$$$$%Ȁ􁝕 ɽAAɕࠤ($$$$$%եե($$$$%($$$%($$%(($$%ե䠤($$$%ȹх]Ѡ ɽAAɕࠤ($$$$%%ɥٽєͥչѼЁUU%ɽȈ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%($$$% U)($$$$%%ɥٽєɽȀͥȁ͸Ёɔ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%($$$% UU%1($$$$%ѼЁեȀȤ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%(($$$%՜эUU%ͥչ́($$$%UU%Ԁձ($$$%($$$$% =5($$$$$%Ԁ􁙕эUU%Ȥ($$$$%($$$%􁍅эፕѥ($$$$% ՜($$$$$%ɥMхQɅ($$$$%($$$%($$$%Ԁձ($$$$%՜ѼЁեȀȤ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%($$$%եԹѽMɥ($$%(($$$9ɵ锁UU%ɥͥ($$%($$$%եձե䠤եՅ%ɕ ͔ձ($$$$%եUU%ɽMɥեɥѽMɥ($$$%($$%􁍅эፕѥɕ($$$$ɔ($$%(($$%Ȁ􁝕Aɽ9եȤ(($$$ хє͕ٕȁ%5A=IQ9PȁɽЁɕѹ̤($$%=A=YѕIѥȤ($$%MɥMٕȀ= ɕAMٕYѕIѥȤ聹ձ($$%ѥՕՕQՕՕQ($$$$$1ѕQܠii%ѕձРѽ%хРѽ5(($$%MMɥɽQɝ̀ՕՕYєձ܁1!͡M($$$$$聹܁1!͡MՕՕYє ɽQɝ̠($$%MMɥɽ݅ɑMٕ̀ՕՕYєձ܁1!͡M($$$$$聹܁1!͡MՕՕYє ɽ݅ɑMٕ̠($$%ɽ ɽ!ՕՕYєձՕՕYєAɽ ɽ!($$%ɽ͕Qх̀􁝕 AɥMٕȠ 5ձѥAɽMР($$%Qх̀ɽ͕Qх̀ չ5Qх̠($$%YѕMх ɽЀ􁍅݅ɑMх ɽСQх̤($$%Ʌ1 յфձ($$%ՕՕQ 􁙅͔(($$$ѥݥѽх́ɕ䁽ȁՕՕٽѕ̸Iո($$$ɔѡ́ٽє́х͔͹͡Ёͼѡձѥ́܁͔($$$ѡеɽٕȁхє($$% хх!ȠQ !($$$%х!Ƞ͡Q ̠($$$%ՕՕQ ѥEՕՔх!ȠQ !($$%(($$$Yєѡٽєɔ䁥єչиQ́́є($$$ٽѕ́ɕѕѡ䁍ЁѡфɽٕȁՕՔ($$$ɕٕ́չٽєѡЁݥЁɽ͕($$%Qх̤($$$%Aɽ5ME0ձ($$$$%Mٕɔ5Ű́Ёɕѱ䰁ѽٽєɽͥ($$$$%ɕɸEՕՕYѕIձйIQId($$$%(($$$%Aɽ5ME0х-EՕե($$$$%Aɽ5ME0єեA9܁хYՕMɥȤ($$$$%Aɽ5ME0Uե̠ե($$$%(($$$%ф􁝕Aɽ5ME0ᅍEՕ䡹܁ յե܁хYՕMɥե($$$%YѕեȰ͕٥фՕՕYєձ($$$$%Yє䁥́ЁЁȀȀ͕٥ͭٽє($$$$%͕YѕIѕȰե͕٥=MٕȤ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%($$%(($$$݅ɐѕɽЁɔѡѥѥٔф($$$ՕՕ́ѡɕ݅ɐѽх́ݽɬQՕՕٕхєɕٕ($$$ɕ她ɽ́ѡЁɕɕ($$%ՕՕQ ($$$%YѕQхM͡ЁɽѕQх̀􁵅Qх̀AɽѕIٕQх̡фȤѕ($$$%YѕMх ɽЀɽ ɽȹ͕%ѕ݅ɑ=($$$$%ɽQɝ̹ɽ ɽȹɕͽٕQɝ̡͔ձ($$$$%ɽ ɽ!Ք($$$%($$$%YѕQEՕՔ啑Yє􁹕܁YѕQEՕՔٽѕ%Ȱ͕٥ѥ($$$$$%ɽ ɽ!ɽQɝ̰ɽ݅ɑMٕ̰($$$$$%ɽѕQх̀ձɽѕQх̹ѽMɥ͔ե($$$%Yѕ !ȠQYѕQ 啑Yє($$$$%MٕɔUѼͥЁՕՕɽٕȁٽєȀȀ͕٥($$$$$$$ͭɽ䁉ɽЈ($$$$%ɕɸEՕՕYѕIձйIQId($$$%($$$%ɽ ɽ!($$$$%ȀMɥхɝЀ聉ɽQɝ̤($$$$$%MMɥ݅ɑ͕Aɽ ɽС ѥ̹ͥѽхɝФեȰ($$$$$$$%͕٥ѥɽѕQх̀ձɽѕQх̹ѽMɥ͔($$$$$%啑Yє ɽ݅ɑMٕ̠݅ɑ($$$$$$%ɽ݅ɑMٕ̹݅ɑ($$$$$$%ͥQYѕٕ䡑啑Yє($$$$$%($$$$%($$$%($$$% ٽєɽȀ͕٥($$$$$$͔ѥ́ɥЁ܈($$$%ɕɸEՕՕYѕIձйMU ML($$%(($$%YѕA䠤(($$$Qх́ɽͥɥ͕ٕȁ=Hձѥɽ($$%ɽ͕Qх̤($$$%Qх̤($$$$%ЁQQх􁝕YՔфQQхĤ($$$$%ЁѡQх􁝕YՔф5ѡQхĤ(($$$$%Ёѕ5ѡQх($$$$% Mѽɕ5ѡQх]ѡє($$$$$% U͕5ѡѕQхAɥQх($$$$$$%ѕ5ѡQх􁝕YՔф5ѡQх]ѡѕAѠĤ($$$$$%􁕱͔($$$$$$%ѕ5ѡQх􁵽ѡQх($$$$$%($$$$%(($$$$%ЁݕQх􁝕YՔф]QхĤ($$$$%ЁQх􁝕YՔфQхĤ($$$$%Ё̀􁝕YՔфÄ A=Yє(($$$$%ЁYѕ̀􁝕 5չ=YѕA䠤($$$$%Yѕ̀($$$$$%1ѕQQ􁝕 չQ ȠQ($$$$$%Ё̀Q=5Ѡ($$$$$%ѡQх̀Yѕ̤($$$$$$%ѡQх􁑅̀Yѕ($$$$$%($$$$%(($$$$% 1YѕÀ̠ 1YѕA̠($$$$$%̀􁝕 1YѕA̠($$$$%(($$$$%ѕЀ􁹕܁YѕQхM͡СQQхѡQхݕQхQх̰($$$$$$%ٽѕAYѕ̰ɕYѕAYѕIեɕѕ5ѡQх(($$$$%Ʌ1 յє􁹕܁Ʌ1($$$$%є܁ յQQх܁хYՕ%СQQх($$$$%є܁ յ5ѡQх܁хYՕ%СѡQх($$$$% Mѽɕ5ѡQх]ѡє($$$$$%є܁ յ5ѡQх]ѡѕAѠ܁хYՕ%Сѕ5ѡQх($$$$%($$$$%є܁ յ]Qх܁хYՕ%СݕQх($$$$%є܁ յQх܁хYՕ%СQх($$$$%є܁ յÄ܁хYՕ%С̤(($$$$%՜Mѥѽх̀ѕйѽMɥٽѕ%􈀬ٽѕ%ȀȀ($$$$$$$͕٥($$$$%Aɽ5ME0єեє($$$%􁕱͔($$$$%ѕЀ􁹕܁YѕQхM͡РٽѕAYѕ̰ɕYѕAYѕIեɕ($$$%($$%($$%ѕЀձ($$$%ѕЀ􁹕܁YѕQхM͡РٽѕAYѕ̰ɕYѕAYѕIեɕ($$%(($$%Yѕ1Mх́ٽѕMх̀Yѕ1Mх̹%55%Q($$%хAɽ ɽЀ􁍅YѕMх ɽЀɽ ɽ!($$$$%ɽ ɽȹ͕%ѕ݅ɑ=($$%MMɥɽ ɽQɝ̀ ѥ̹MР($$%хAɽ ɽФ($$$$ՕՕɽЁ݅́ͅɥͅݡѡȁ݅($$$$I䁽хɝ́ѡЁЁɕ٥ͱ䁅Ёٕ($$$%ɽ ɽQɝ̀ɽ ɽ!܁1!͡MɽQɝ̤($$$$$$ɽ ɽȹɕͽٕQɝ̡͔ձ($$$%MMɥɕQɝ̀􁹕܁1!͡Mɽ ɽQɝ̤($$$%ɕQɝ̹ɕٕɽ݅ɑMٕ̤($$$%ɽ݅ɑMٕ̹͕Aɽ ɽСɕQɝ̰եȰ͕٥ѥ($$$$$%ѕЀձѕйѽMɥ͔($$%(($$$($$$Mٽє̤Ѽ̤($$$($$% MYѕQMٕ̠($$$%ȀMɥ̀聝مMٕ̠(($$$$%ɍ 􁝕 ]U͕=($$$$$$$=MٕȀձMٕȹՅ%ɕ ͔̤(($$$$%ɍ ($$$$$%՜ɍٽєѼȁ͕ٕȀ̤($$$$%(($$$$%M=MٕYѕIѥ̤ѡɕեɕA=ɍ ($$$$$%ٽѕMх̀Yѕ1Mх̹  !($$$$$%ɽ݅ɑхAɽ ɽ($$$$$$$$ɽ݅ɑMٕ̹хɽ ɽQɝ̤($$$$$%Yѕ !ȠMٕYє̰($$$$$$$%܁= չYєٽѕ%Ȱե͕٥ѥɕYє($$$$$$$$$%ѕйѽMɥɽ݅ɑхAɽ ɽа($$$$$$$$$%ɽ ɽQɝ̰ɽ݅ɑMٕ̰͔($$$$$%՜ ٽєȀȀ͕٥Ȁ̤($$$$%􁕱͔($$$$$%ɽ!ɔ􀅉ɽ݅ɑMٕ̹х̡̤($$$$$%ɽ!ɔ Aɽ ɽ($$$$$$%MMɥхɝ̀хAɽ ɽЀɽ ɽQɝ($$$$$$$$$ɽ ɽȹɕͽٕQɝ̡=MٕȤ($$$$$$%ɽ!ɔɽ ɽȹ͡ձ ɽС̰хɝ̤($$$$$%(($$$$$%5ͅAɽ!ȹ͕5̰ͅȰ($$$$$$$%YѥA՝]ɔٽєȰե͕٥ѥՔɕYєѕйѽMɥ($$$$$$$$$%ٽѕ% չ5Qх̠ɽ!ɔİĤ($$$$%($$$%($$%􁕱͔($$$$M͕ٕȁ聽́Ѽȁ͕ٕ쁽ѡݥ͔ՕՔ̀($$$$ٽє($$$%=MٕȀձمMٕ̠х̡MٕȤ($$$$%Mɥ͕ٕȀMٕ(($$$$%ɽ!ɔ􀅉ɽ݅ɑMٕ̹х̡͕ٕȤ($$$$%ɽ!ɔ Aɽ ɽ($$$$$%MMɥхɝ̀хAɽ ɽЀɽ ɽQɝ($$$$$$$$ɽ ɽȹɕͽٕQɝ̡ՔMٕȤ($$$$$%ɽ!ɔɽ ɽȹ͡ձ ɽС͕ٕȰхɝ̤($$$$%(($$$$%5ͅAɽ!ȹ͕5͕ٕͅȰİ($$$$$$%YѥA՝]ɔٽѕ=Ȱե͕٥ѥՔɕYєѕйѽMɥ($$$$$$$$%ٽѕ% չ5Qх̠ɽ!ɔİĤ(($$$$%YѕMх ɽЀ Aɽ ɽ($$$$$$$хAɽ ɽФ($$$$$%MMɥхɝ̀ɽ ɽȹɕͽٕQɝ̡ՔMٕȤ(($$$$$%Ё($$$$$%ȀMɥхɝMٕȀхɝ̤($$$$$$$ٽՉɽЁѡ͕ٕͅȁѡЁɕ䁝Ёѡٽѕ=($$$$$$%хɝMٕȹՅ%ɕ ͔͕ٕȤ($$$$$$$%ѥՔ($$$$$$%($$$$$$% Mٕ̠х̡хɝMٕȤ($$$$$$$%ѥՔ($$$$$$%(($$$$$$%5ͅAɽ!ȹ͕5ͅхɝMٕȰ($$$$$$$$%YѥA՝]ɔٽѕ ɽСեȰ͕٥ѥ($$$$$$$$$$%ѕЀձѕйѽMɥՔ($$$$$$%䬬($$$$$%($$$$%(($$$$$ձѥɽ聕ٕ䁍ȁٽє($$$$% 5ձѥAɽMР 5ձѥAɽ=I݅ɐ($$$$$%ձѥAɽ!ȹ͕ YєեȤ($$$$%($$$%􁕱͔($$$$%ٽѕMх̀Yѕ1Mх̹  !($$$$%ɽ݅ɑхAɽ ɽ($$$$$$$ɽ݅ɑMٕ̹хɽ ɽQɝ̤($$$$%Yѕ !Ƞ=Yєե($$$$$$%܁= չYєٽѕ%Ȱե͕٥ѥɕYєѕйѽMɥ($$$$$$$$%ɽ݅ɑхAɽ ɽаɽ ɽQɝ̰($$$$$$$$%ɽ݅ɑMٕ̰͔($$$$%՜ ٽєȀȀ͕٥($$$%(($$$%Ё($$$%ȀMɥ̀聝مMٕ̠($$$$%5ͅAɽ!ȹ͕5̰ͅ䀬İYѥA՝]ɔٽѕUєե($$$$$$%ٽѕAYѕ̰ɕYѕAYѕIեɕ͕٥ѥѕйѽMɥ($$$$%䀬($$$%($$%(($$$Yє($$%ٽѕ15űQձ Yѕ1($$$%ٽѕ15űQYєٽѕ%ٽѕMх̰͕٥եȰѥ($$$$$%Yѕ !ȠAɽ Qхե($$%(($$$($$$5ձѥɽ䁙݅ɑ($$$($$% 5ձѥAɽMР AɥMٕȠ($$$% 5ձѥAɽ=I݅ɐ($$$$%՜Mɽٽєٕ($$$$%ձѥAɽ!ȹ͕5ձѥAɽٕYѥA՝]ɔٽєȰե͕٥ѥ͔($$$$$$%ɕYєѕЀձѕйѽMɥٽѕ%͔͔İĤ($$$%􁕱͔($$$$$=͕Ѽѡȁɽᥕ́ѡȁ%9=Pɕɕٔɕ݅ɐ($$$$$($$$$%͡ձMՔ($$$$%=MٕȀձ($$$$$% Mٕ̠х̡MٕȤ($$$$$$%͡ձM􁙅͔($$$$$%($$$$%(($$$$%͡ձM($$$$$%՜Mɽٽѕٕ($$$$$%ձѥAɽ!($$$$$$$$͕5ձѥAɽٕYѥA՝]ɔٽѕ=Ȱե͕٥ѥ͔($$$$$$$$$%ɕYєѕЀձѕйѽMɥٽѕ%͔͔İĤ($$$$%􁕱͔($$$$$%՜9Ё͕ɽ䁵ͅȁٽѕȁɕ䁝Ёɕ݅ɐ($$$$%($$$%($$%($$%ՕՕYєձ($$$%ՕՕYє͕Aɽ͕Ք($$$%Yѕ !ȠѕQYєՕՕYє($$$$%݅ɸUѼͥЁѕɽٕȁٽєՕՕYєYѕ%($$$$$$$쁅ѕѥɅɕمѕ䈤($$$%($$%($$%ɕɸEՕՕYѕIձйMU ML($%􁍅эፕѥ($$%ɥMхQɅ($$%ɕɸEՕՕYѕIձйIQId($%(%((%ɥمєхѥ́AAɕ͕!($%ɥمєUU%ɕՕ%($%ɥمєUU%Uե($%ɥمєMɥ9($%ɥمєMɥե($%ɥمєMɥ͕ٕ($%ɥمєUU%ѥ%($%ɥمєUU%%ɹѥ%($%ɥمєMхѕ($%ɥمєMՕ($%ɥمєɕѕ(($%ɥمєAAɕ͕!Mɥ9MɥեMɥ͕ٕȰUU%ѥ%($$$%UU%%ɹѥ%MхѕаMՕɕѕФ($$%ѡ̹Uե͕AUեե($$%ѡ̹99($$%ѡ̹եե($$%ѡ̹͕ٕȀ͕ٕ($$%ѡ̹ѥ%􁍽ѥ%($$%ѡ̹%ɹѥ%􁉅%ɹѥ%($$%ѡ̹MхѕЀ􁉅Mхѕ($$%ѡ̹MՕ􁍽MՕ($$%ѡ̹ɕѕЀɕѕ($%(($%ɥمєхѥUU%͕AUեMɥե($$%($$$%ɕɸUU%ɽMɥեɥ($$%􁍅эፕѥɕ($$$%ɕɸձ($$%($%(%((%ՉɅЁٽ݅ɸMɥͅ((%ՉɅЁMձᕍѽM٥MձȠ)( \ No newline at end of file diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index 8ebd3b1496..9cc07ce0fb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -478,27 +478,30 @@ public String getMqttPrefix() { return getData().getString("MQTT.Prefix", ""); } - public void load() { - if (!bungee.getDataFolder().exists()) { - bungee.getDataFolder().mkdir(); + public void load() { + try { + loadControlConfiguration(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + /** Loads the configuration while propagating failures to Control's rollback path. */ + public void loadControlConfiguration() throws IOException { + if (!bungee.getDataFolder().exists()) { + bungee.getDataFolder().mkdir(); } - File file = new File(bungee.getDataFolder(), "bungeeconfig.yml"); - - if (!file.exists()) { - try (InputStream in = bungee.getResourceAsStream("bungeeconfig.yml")) { - Files.copy(in, file.toPath()); - } catch (IOException e) { - e.printStackTrace(); - } - } - try { - data = ConfigurationProvider.getProvider(YamlConfiguration.class) - .load(new File(bungee.getDataFolder(), "bungeeconfig.yml")); - } catch (IOException e) { - e.printStackTrace(); - } - } + File file = new File(bungee.getDataFolder(), "bungeeconfig.yml"); + + if (!file.exists()) { + try (InputStream in = bungee.getResourceAsStream("bungeeconfig.yml")) { + Files.copy(in, file.toPath()); + } + } + data = ConfigurationProvider.getProvider(YamlConfiguration.class) + .load(new File(bungee.getDataFolder(), "bungeeconfig.yml")); + } public void save() { try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index 4a42a1cdff..9c729f0436 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -489,9 +489,14 @@ public void debug(String str) { } @Override - public Set getAllAvailableServers() { - return getAvailableAllServers(); - } + public Set getAllAvailableServers() { + return getAvailableAllServers(); + } + + @Override + public Set getAllConfiguredServers() { + return new HashSet<>(getProxy().getServers().keySet()); + } @Override public VotingPluginProxyConfig getConfig() { @@ -702,10 +707,16 @@ public void warn(String message) { } @Override - public void reloadCore(boolean mysql) { + public void reloadCore(boolean mysql) { // mysql==true should do full reloadall behavior on the platform - reloadPlugin(mysql); - } + reloadPlugin(mysql); + } + + @Override + public void reloadControlConfiguration() throws Exception { + config.loadControlConfiguration(); + reloadFromControl(); + } @Override public ScheduledExecutorService getScheduler() { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java index 835e6d40c8..9c9014d845 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java @@ -9,20 +9,35 @@ /** Validates, persists, reloads, and rolls back the bounded Control configuration domain. */ public final class ProxyRoutingConfigurationService { - private final VotingPluginProxy proxy; + private final Platform platform; public ProxyRoutingConfigurationService(VotingPluginProxy proxy) { - this.proxy = proxy; + this(new Platform() { + @Override public ProxyRoutingConfiguration read() { + return new ProxyRoutingConfiguration(proxy.getConfig().getSendVotesToAllServers(), + proxy.getConfig().getBlockedServers()); + } + @Override public Set configuredServers() { return proxy.getAllConfiguredServers(); } + @Override public void persist(ProxyRoutingConfiguration proposal) throws IOException { + proxy.getConfig().persistControlProxyRouting(proposal.sendVotesToAllServers(), + proposal.blockedServers()); + } + @Override public void rollback() throws IOException { proxy.getConfig().rollbackControlProxyRouting(); } + @Override public void reload() throws Exception { proxy.reloadControlConfiguration(); } + }); + } + + ProxyRoutingConfigurationService(Platform platform) { + this.platform = platform; } public ProxyRoutingConfiguration read() { - return new ProxyRoutingConfiguration(proxy.getConfig().getSendVotesToAllServers(), - proxy.getConfig().getBlockedServers()); + return platform.read(); } public void validate(ProxyRoutingConfiguration proposal) { Set available = new HashSet<>(); - proxy.getAllAvailableServers().forEach(server -> available.add(server.toLowerCase(Locale.ROOT))); + platform.configuredServers().forEach(server -> available.add(server.toLowerCase(Locale.ROOT))); for (String server : proposal.blockedServers()) { if (!available.contains(server.toLowerCase(Locale.ROOT))) { throw new IllegalArgumentException("unknown backend: " + server); @@ -35,14 +50,14 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t if (expectedRevision == null || !read().revision().equals(expectedRevision)) { throw new StaleRevisionException(); } - proxy.getConfig().persistControlProxyRouting(proposal.sendVotesToAllServers(), proposal.blockedServers()); + platform.persist(proposal); try { - proxy.reloadCore(false); - } catch (RuntimeException e) { + platform.reload(); + } catch (Exception e) { boolean rolledBack = false; try { - proxy.getConfig().rollbackControlProxyRouting(); - proxy.reloadCore(false); + platform.rollback(); + platform.reload(); rolledBack = true; } catch (Exception rollbackFailure) { e.addSuppressed(rollbackFailure); @@ -51,6 +66,14 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t } } + interface Platform { + ProxyRoutingConfiguration read(); + Set configuredServers(); + void persist(ProxyRoutingConfiguration proposal) throws IOException; + void rollback() throws IOException; + void reload() throws Exception; + } + @SuppressWarnings("serial") public static final class StaleRevisionException extends RuntimeException { } @SuppressWarnings("serial") diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 5c2d550601..904c9fad61 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -62,7 +62,13 @@ public synchronized void rollbackControlProxyRouting() throws IOException { Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); atomicReplace(stage, target); - reload(); + loadControlConfiguration(); + } + + /** Loads the active file without the superclass's empty-config fallback. */ + public synchronized void loadControlConfiguration() throws IOException { + ConfigurationNode loaded = YamlConfigurationLoader.builder().path(configurationFile.toPath()).build().load(); + setConf(loaded); } private static void atomicReplace(Path source, Path target) throws IOException { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 693f4f48d3..6d6a613ac6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -577,6 +577,13 @@ public Set getAllAvailableServers() { return getAvailableAllServers(); } + @Override + public Set getAllConfiguredServers() { + Set configured = new HashSet<>(); + server.getAllServers().forEach(registered -> configured.add(registered.getServerInfo().getName())); + return configured; + } + @Override public VotingPluginProxyConfig getConfig() { return config; @@ -802,6 +809,12 @@ public void reloadCore(boolean mysql) { reloadAllInternal(mysql); } + @Override + public void reloadControlConfiguration() throws Exception { + config.loadControlConfiguration(); + reloadFromControl(); + } + @Override public ScheduledExecutorService getScheduler() { return timer; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java new file mode 100644 index 0000000000..9037f5e076 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java @@ -0,0 +1,56 @@ +package com.bencodez.votingplugin.proxy.control; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.io.IOException; +import java.util.List; +import java.util.Set; + +import org.junit.jupiter.api.Test; + +class ProxyRoutingConfigurationServiceTest { + @Test + void validatesBlockedServersAgainstCompleteConfiguredSet() { + FakePlatform platform = new FakePlatform(); + platform.configuredServers = Set.of("Lobby"); + + assertDoesNotThrow(() -> new ProxyRoutingConfigurationService(platform) + .validate(new ProxyRoutingConfiguration(true, List.of("lobby")))); + } + + @Test + void strictReloadFailureRestoresBackupAndReloadsAgain() throws Exception { + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + ProxyRoutingConfiguration proposal = new ProxyRoutingConfiguration(true, List.of()); + FakePlatform platform = new FakePlatform(); + platform.current = current; + platform.failFirstReload = true; + + ProxyRoutingConfigurationService.ApplyFailureException failure = assertThrows( + ProxyRoutingConfigurationService.ApplyFailureException.class, + () -> new ProxyRoutingConfigurationService(platform).apply(proposal, current.revision())); + + assertTrue(failure.rolledBack()); + assertTrue(platform.rolledBack); + assertTrue(platform.reloadCalls == 2); + } + + private static final class FakePlatform implements ProxyRoutingConfigurationService.Platform { + private ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + private Set configuredServers = Set.of(); + private boolean failFirstReload; + private boolean rolledBack; + private int reloadCalls; + + @Override public ProxyRoutingConfiguration read() { return current; } + @Override public Set configuredServers() { return configuredServers; } + @Override public void persist(ProxyRoutingConfiguration proposal) { current = proposal; } + @Override public void rollback() { rolledBack = true; } + @Override public void reload() throws Exception { + reloadCalls++; + if (failFirstReload && reloadCalls == 1) throw new IOException("invalid reloaded configuration"); + } + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java index 3814ddbfcf..2bfbe1e7c6 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginProxyTestImpl.java @@ -45,6 +45,11 @@ public Set getAllAvailableServers() { return new HashSet<>(Arrays.asList("Server1", "Server2")); } + @Override + public Set getAllConfiguredServers() { + return getAllAvailableServers(); + } + @Override public VotingPluginProxyConfig getConfig() { if (config == null) { @@ -160,6 +165,11 @@ public void reloadCore(boolean mysql) { // Mocked for testing } + @Override + public void reloadControlConfiguration() throws Exception { + // Mocked for testing + } + @Override public void log(String message) { // For testing, simply print the message diff --git a/docs/control-connector.md b/docs/control-connector.md index 1f8b9d3b13..27859c74b7 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -60,10 +60,16 @@ current release is retained as `.previous`. A failed process/protocol health che previous release, retaining the failed candidate as `.failed`. Unexpected exits use bounded restart backoff. The child receives only its bind host, port, and contained data directory. It does not receive VotingPlugin configuration -or credentials in release metadata. Process output is written beside the hosted JAR and rotated at 1 MiB. The WebUI is -then available at `http://127.0.0.1:8080/` and asks for Control's separate admin token. To expose a non-loopback listener, -create the admin token in `Control.Hosted.DataDirectory` first and use HTTPS or a private tunnel; Control otherwise refuses -the bind. +or credentials in release metadata. Process output is written beside the hosted JAR and rotated at 1 MiB. Configure the +WebUI password directly against the hosted data directory (the prompt is not echoed and the password is not an argument): + +```shell +java -jar plugins/VotingPlugin/control/votingplugin-control.jar web-password plugins/VotingPlugin/control/data +``` + +The WebUI is then available at `http://127.0.0.1:8080/`. Password rotation invalidates every existing browser session. +To expose a non-loopback listener, create either the WebUI password or an admin API token in +`Control.Hosted.DataDirectory` first and use HTTPS or a private tunnel; Control otherwise refuses the bind. Manual installation remains supported: put a Control JAR at `JarFile`, configure its exact SHA-256, set `AutoDownload` and `AutoUpdate` false, and enable hosting. VotingPlugin will verify and supervise it without network access. From dcc15c548eb519d07ec064c6cdac5ff861a003a9 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 24 Aug 2026 20:40:33 -0600 Subject: [PATCH 06/73] Restore complete proxy source --- .../votingplugin/proxy/VotingPluginProxy.java | 3262 ++++++++++++++++- 1 file changed, 3261 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index a6e507f63b..e51b4e4f66 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1 +1,3261 @@ -Yx-jםi+j[hܢͶ:-jZ.)޳W6vR6&V6FWfFwVv&'Bf'FT'&WGWE7G&VӰ'BfFFWE7G&VӰ'BfFFWGWE7G&VӰ'BffS'BfW6WF㰦'BfWBWE66WDFG&W73'BfWB66WC'BfWBU$'BfWBGGGG6ƖVC'BfWBGGGG&WVW7C'BfWBGGGG&W76S'Bf75W6WF㰦'BfFRGW&F㰦'BfFR7FC'BfFR6FFUFS'BfFRTC'BfFRTfg6WC'BfWF'&Ɨ7C'BfWF6V7F㰦'BfWF6V7F3'BfWF6'BfWFƖVD66WC'BfWFƗ7C'BfWF'BfWF6WC'BfWFUTC'BfWF67W'&VB67W'&VD6'BfWF67W'&VB66VGVVDWV7WF%6W'f6S'BfWF67W'&VBFUVC'B&rV6Ɨ6R6ƖVBGGc2GDW6WF㰠'B6&V6FWGf6VF6&RFRFUGS'B6&V6FWGf6VF6&R'VvVVv&FFv&FFFW%&'B6&V6FWGf6VF6&R'VvVVv&FFv&ו5ð'B6&V6FWGf6VF6&R'VvVVFR'VvVUFT6V6W#'B6&V6FW6VV7'FV7'FFW#'B6&V6FW6V66'6W#'B6&V6FW6V6W'fW&66FV26VfVS'B6&V6FW6V6W'fW&66FV26VfVT6FV3'B6&V6FW6V6W'fW&6v&v&W76vTƗ7FVW#'B6&V6FW6V6W'fW&6v&v&W76vU&FW#'B6&V6FW6V6W'fW&6GBGDFW#'B6&V6FW6V6W'fW&6GBGE6W'fW$6Ӱ'B6&V6FW6V6W'fW&6ח7ו7W76VvW#'B6&V6FW6V6W'fW&6&VF2&VF4FW#'B6&V6FW6V6W'fW&6&VF2&VF4Ɨ7FVW#'B6&V6FW6V6W'fW&666WG26ƖVDFW#'B6&V6FW6V6W'fW&666WG266WDFW#'B6&V6FW6V6W'fW&666WG266WE&V6VfW#'B6&V6FW6V76V㰦'B6&V6FW6V7FFGS'B6&V6FW6V7FFFFfVS'B6&V6FW6V7FFFFfVT&V㰦'B6&V6FW6V7FFFFfVTC'B6&V6FW6V7FFFFfVU7G&s'B6&V6FW6V7ח76frח76fs'B6&V6FWfFwVv&'&F67B&'&F67DFV6FW#'B6&V6FWfFwVv&66RfFT66S'B6&V6FWfFwVv&66RfFT66TFW#'B6&V6FWfFwVv&66RfFVBfFVEW'57F&vS'B6&V6FWfFwVv&66RfFVBfFVEW'466S'B6&V6FWfFwVv&6G&6G&6V7F#'B6&V6FWfFwVv&6G&7FVD6G&vW#'B6&V6FWfFwVv&VF&VF&FW#'B6&V6FWfFwVv&VF&VF&WFC'B6&V6FWfFwVv&VF&VF&6W'fW%66WD6fwW&F㰦'B6&V6FWfFwVv&VF&VF&6W'fW%66WD6fwW&F'VvVS'B6&V6FWfFwVv&&W6V6R&6VEW%&W6V6UG&6W#'B6&V6FWfFwVv&&W6V6RW%&W6V6S'B6&V6FWfFwVvFWVWVRfFUFUVWVS'B6&V6FWfFwVvFfFW"FfFW#'B6&V6FWfFwVvWF֖V7&gEW6W&UfƖFF#'B6&V6FWfFwVvWF6W'f6U6FUfƖFF#'B6&V6FWfFwVvfFVrfFTtח7F&S'B6&V6FWfFwVvfFVrfFTtח7F&RfFTu7FGW3'B6vvRw66VVVC'B6vvRw66&V7C'B&VF26ƖVG2VF2FVfVDVF46ƖVD6fs'B&VF26ƖVG2VF27DE'C'B&VF26ƖVG2VF2VF3'B&VF26ƖVG2VF2VF5ð'B&vWGFW#'B&6WGFW#V&Ɩ2'7G&7B672fFuVv&&fFR7FF2fr$U4T4UDdeDTUEԔĕ2FUVBԔUDU2F֖Ɨ2"&fFR7FF2fr$U4T4U5D%EU$U55DT4T4E2Tð&fFR7FF2fr$U4T4UDT4UDU%d4T4E23ð&fFR7FF2fr$U4T4U$4TEDTUEԔĕ2FUVB4T4E2F֖Ɨ2vWGFW 6WGFW &fFRBfFU'GfFW2vWGFW 6WGFW &fFRB7W'&VEfFU'GfFW5&WV&VBvWGFW 6WGFW &fFR&ח7W6W%F&R&ו5ð&fFRV7'FFW"V7'FFW#&fFR67G&r6ƖVDFW#6ƖVDFW3&fFR66WDFW"66WDFW#vWGFW 6WGFW &fFR&VfFfW$V&VBG'VSvWGFW &fFR67W'&VD6UTB7G&sWVEW$T66RWr67W'&VD6ₓvWGFW 6WGFW &fFRv&FFFW%&v&FFFW#vWGFW &fFR&VF4FW"&VF4FW#&fFRVF5&VF5V&Ɨ6W%ð&fFRfFRr&VF5V&Ɨ6W%&WG'gFW#&fFR&VFUfFU&WG'66VGVVC&fFR&VFUfFTFVƗfW'&WG'66VGVVC&fFR&V66VEfFTFVƗfW'&WG'66VGVVC&fFR&VV&VCvWGFW 6WGFW &fFRVF&FW"VF&FW#vWGFW &fFR'VvVUFT6V6W"'VvVUFT6V6W#vWGFW 6WGFW &fFR'VvVTWFBWFCvWGFW &fFRGDFW"GDFW#vWGFW &fFRv&W76vU&FW"v&W76vU&FW#vWGFW 6WGFW &fFRו7W76VvW"&ח7W76VvW#vWGFW &fFRfFT66TFW"fFT66TFW#vWGFW &fFRfFVEW'466RfFVEW'466SvWGFW &fFRf&6VEW%&W6V6UG&6W"&6VEW%&W6V6UG&6W"Wr&6VEW%&W6V6UG&6W"&fFRfUTBVFu&W6V6TFfcVFu&W6V6TFfg2Wr6ₓ&fFRf6WC7G&sVFt&6VE&V6fW'66G267W'&VD6WtW6WB&fFRfFR6G&6V7F"6G&6V7F#&fFRfFR7FVD6G&vW"7FVD6G&vW#V&Ɩ2fFuVv&V&VBG'VS'VvVUFT6V6W"Wr'VvVUFT6V6W"vWD6frvWEFURvWD6frvWEFTW$fe6WBvWD6frvWEFUvVVfe6WBfW'&FPV&Ɩ2fBFV'Vr7G&rFWBFV'Vs"FWBРfW'&FPV&Ɩ2rvWD7EWFFVB&WGW&vWEfFT66T7EWFFVBРfW'&FPV&Ɩ2BvWE&WdF&WGW&vWEfFT66U&WdFРfW'&FPV&Ɩ27G&rvWE&WdF&WGW&vWEfFT66U&WdFРfW'&FPV&Ɩ2BvWE&WevVV&WGW&vWEfFT66U&WevVVРfW'&FPV&Ɩ2fBf7G&rFWBrFWBРfW'&FPV&Ɩ2&V4V&VB&WGW&V&VCРfW'&FPV&Ɩ2&V4v&UFR&WGW&5fFT66Tv&UFRРfW'&FPV&Ɩ2fB6WDv&UFR&Vv&R6WEfFT66UfFT66Tv&UFRv&RРfW'&FPV&Ɩ2fB6WD7EWFFVB6WEfFT66T7EWFFVBРfW'&FPV&Ɩ2fB6WE&WdFBF6WEfFT66U&WdFFРfW'&FPV&Ɩ2fB6WE&WdF7G&rFWB6WEfFT66U&WdFFWBРfW'&FPV&Ɩ2fB6WE&WevVVBvVV6WEfFT66U&WevVVvVVРfW'&FPV&Ɩ2fBFT6vVBFUGRGR&VfR&V&R&V7BbvWD6frvWEfFT66UFRvWEfFT66TFW"6V6fFT66UFRvWD6frvWEfFT66UFRРbvWD6frvWDv&FFV&VBv&$v&FFBV&VBv&rFR6vRWfVB"&WGW&㰠РBFVf"7G&r2vWDf&U6W'fW'2bvWDv&FFFW"vWDv&ח7‚6F4W27G&r7DƖU7G"vWDv&FFFW"vWE7G&r2$7DƖR"r7DƖRG'7DƖRrfVTb7DƖU7G"6F6V&W$f&DW6WFRv&PРb6FFUFRrERTfg6WBUD2F7FBFW6֖ƒ7DƖRcc"67G&rFFfVSFFF6WBWr6ₓFFF6WBWB$7EWFFVB"WrFFfVU7G&r""6FFUFRrERTfg6WBUD2F7FBFW6֖ƒFFF6WBWB$f6VE&6W76r"WrFFfVT&Vf6RFFF6WBWBGRF7G&rWrFFfVT&VG'VRvWDv&FFFW"6WDFF2FFF6WBv&W76vU&FW"6VDW76vR2FVfFuVvv&R'VvVUFT6vRFVV6Rv&%6W'fW""2"6wB&VVƖR&V6VFǒ"РV6Rv&%6W'fW""2"v&FFFW"F6&VC"РРv&FFFW"FT6vRGRРfW'&FPV&Ɩ2fBv&r7G&rFWBv&FWBРӰРV&Ɩ2fBFT6vVDfVB7G&r7'bFUGRGRvWDv&FFFW"6WD&V7'bGRF7G&rf6RvWDv&FFFW"6WD&V7'b$f6VE&6W76r"G'VRvWDv&FFFW"6WD&V7'b%&6W76r"f6RРV&Ɩ2fBFT6vVDf6VBFUGRGRbGRWV2FUGRDvWE&ו5‚66VFFFfFW"FǒvWD6VR$7DFFF"РvWE&ו5‚vT6VFFFfFW"bGRvWD6VRFFGRDTtU"bvWD6frvWDv&FFV&VB&WGW&㰠Рf"7G&r2vWDf&U6W'fW'2vWDv&FFFW"6WD&V2$f&6UWFFR"G'VRvWDv&W76vU&FW"6VDW76vR2fFuVvv&R'VvVUFT6vRР&6W75VWVRР򢠠Bו5v&FFFW"ࠒV&Ɩ2fBDח7„ח76frח76frח76frv&FFח76frbח76frvWD7DR4VGח76frvWDFF&6R4VGu6WfW&R$ו52B6fwW&VB6'&V7Fǒ"$֗76r7BFF&6R7C"ח76frvWD7DR"F#"ח76frvWDFF&6R6WE&ו5†V“&WGW&㰠Р6WE&ו5†Wr&ח7W6W%F&R%fFuVvW6W'2"ח76frvWD6frvWDFV'VrfW'&FPV&Ɩ2fBFV'Vr5W6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBu6WfW&R7G&r7G&rfFuVv&F2u6WfW&R7G&rРfW'&FPV&Ɩ2fBtf7G&r7G&rfFuVv&F2tf7G&rРfW'&FPV&Ɩ2fBFV'VrF&v&RBbvWD6frvWDFV'VrB&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vr7G&r7G"FV'Vs"7G"Рғ'&Ɨ7C7G&s6W'fW'2Wr'&Ɨ7C7G&svWDf&U6W'fW'2bvWD6frvWDv&FFV&VBbvWD6frvWDv&FFW6Tו5‚6WDv&FFFW"Wrv&FFFW%&Wrv&ו5‚%fFuVvv&FF"vWE&ו5‚vWDח7‚fW'&FPV&Ɩ2fBFV'VtWW6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vtr7G&rFWBFV'VrFWBРfW'&FPV&Ɩ2fBf7G&rFWBtfFWBРfW'&FPV&Ɩ2fBu6WfW&R7G&rFWBfFuVv&F2u6WfW&RFWBРfW'&FPV&Ɩ2fBv&r7G&rFWBv&FWBР6W'fW'2fW'&FPV&Ɩ2fBFT6vVDfVB7G&r7'bFUGRGRfFuVv&F2FT6vVDfVB7'bGRРfW'&FPV&Ɩ2fBFT6vVDf6VBFUGRGRfFuVv&F2FT6vVDf6VBGRРғV6R6WDv&FFFW"Wrv&FFFW%&Wrv&ו5‚%fFuVvv&FF"v&FFח76frfW'&FPV&Ɩ2fBFV'VtWW6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vtr7G&rFWBFV'VrFWBРfW'&FPV&Ɩ2fBf7G&rFWBtfFWBРfW'&FPV&Ɩ2fBu6WfW&R7G&rFWBfFuVv&F2u6WfW&RFWBРfW'&FPV&Ɩ2fBv&r7G&rFWBv&FWBР6W'fW'2fW'&FPV&Ɩ2fBFT6vVDfVB7G&r7'bFUGRGRfFuVv&F2FT6vVDfVB7'bGRРfW'&FPV&Ɩ2fBFT6vVDf6VBFUGRGRfFuVv&F2FT6vVDf6VBGRРғРWFFRv&66V6V2V6vVBg&&vvWDv&FFFW"vWDv&ח7‚FW$6VGR$v&UFR"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$D"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR%tTT"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$D"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$f6VE&6W76r"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR%&6W76r"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$f&6UWFFR"%d$4"R"vWDv&FFFW"vWDv&ח7‚FW$6VGR$7EWFFVB"$TDTDUB"Р6VGW2V6vVBg&&vvWE&ו5‚FW$6VGR%FfFW$v&R"%d$4"R"vWE&ו5‚FW$6VGR$6V6v&B"%d$4"R"vWE&ו5‚FW$6VGR%&V֖FVB"%d$4"R"vWE&ו5‚FW$6VGR$F6&T'&F67B"%d$4"R"vWE&ו5‚FW$6VGR$7DƖR"%d$4"#"vWE&ו5‚FW$6VGR%W$R"%d$4"3"vWE&ו5‚FW$6VGR$FǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR%vVVǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$FfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR$&W7DFfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR%vVVfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR$&W7EvVVfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR%fFU'GfFW2"$BDTdTBsr"vWE&ו5‚FW$6VGR$FfFU7G&V"$BDTdTBsr"vWE&ו5‚FW$6VGR%G2"$BDTdTBsr"vWE&ו5‚FW$6VGR$vW7DFǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$FUFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$vW7DFǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$FFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$vW7EvVVǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$7DFFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$7EvVVǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$7DFǕFF"$BDTdTBsr"vWE&ו5‚FW$6VGR$ffƖU&Wv&G2"$TDTDUB"vWE&ו5‚FW$6VGR$FfFU7G&V7EWFFR"$TDTDUB"bvWD6frvWE7F&TFFF5vFFFRvWE&ו5‚FW$6VGRvWDFFF5vFFFUF6FFUFRr$BDTdTBsr"vWE&ו5‚FW$6VGRvWDFFF5vFFFUF6FFUFRrW4F2$BDTdTBsr"vWE&ו5‚FW$6VGRvWDFFF5vFFFUF6FFUFRrW4F2"$BDTdTBsr"РРV&Ɩ2fBFD7W'&VEfFU'GfFW2BVBfFU'GfFW2VC6WEfFT66UfFU'G7W'&VEfFW2fFU'GfFW2FV'Vr$7W'&VBfFR'GFFâ"fFU'GfFW2РV&Ɩ2fBFDfFVEW"7G&rWVB7G&rW$RfFVEW'466RFEW"WVBW$RРV&Ɩ2fBFEfFU'GbvWD6frvWEfFU'GV&VBFD7W'&VEfFU'GfFW26V6fFU'GРРV&Ɩ2'7G&7BfB'&F67B7G&rW76vR&fFR6WC7G&s6VE&'&F67B6WC7G&sF&vWG27G&rWVB7G&rW"7G&r6W'f6RrFR7G&rFWB&Vv4ƖR6WC7G&sf'v&FVBWrƖVD66WCₓf"7G&rF&vWE6W'fW"F&vWG26VfVRVfVRfFuVvv&RfFT'&F67BWVBW"6W'f6RFRFWBv4ƖRb6VE&'&F67DVfVTrF&vWE6W'fW"VfVRf'v&FVBFBF&vWE6W'fW"РР&WGW&f'v&FVCР򢠠6VG27FFR&'&F67BF&VvFR6VV7FVBG&7'BB&W'G0vWFW"FBG&7'B66WFVBFRW76vRࠒ&6W'fW"F&vWB&6VB6W'fW &VfVR7FFR'&F67BVfVP&WGW&G'VRǒvVFRG&7'B66WFVBFRW76vP&FV7FVB&V6VE&'&F67DVfVTr7G&r6W'fW"6VfVRVfVR7vF6WFB66REC&WGW&6VDGDVfVU6W'fW"6W'fW"VfVR66RՕ5àb&ח7W76VvW"V’&WGW&f6SРG'&ח7W76VvW"6VEF&6VB6W'fW"VfVR&WGW&G'VS6F65W6WFRFV'VrRvWDW76vR&WGW&f6SР66RTtU54ts&WGW&6VEVvW76vU6W'fW$r6W'fW"VfVR66R$TD3&WGW&6VE&VF4VfVU6W'fW"6W'fW"VfVRG'VR66R44UE37FFR'&F67G2W6RFR6RFƗVB6ƖVB2&VfVW2F2&W6W'fW2FR66WB6V7FBG2FVƗfW'6vVFvVVB7FVBb7&VFr6V6B6'BƗfVB66WBࠐ6ƖVDFW"66WD6ƖVB6ƖVDFW2VV6ƖVDFW2vWB6W'fW"b66WD6ƖVBV’&WGW&f6SРG'66WD6ƖVB6VDVfVRVfVR&WGW&G'VS6F6'VFTW6WFRFV'VrRvWDW76vR&WGW&f6SРFVfVC&WGW&f6SРРV&Ɩ276&旦VBfB6V666VEfFW27G&r6W'fW"BFVb56W'fW%fƖB6W'fW"b56VTƖU6W'fW$f%fFU&WFr6W'fW"bvWEfFT66TFW"5fFW26W'fW"bbvWD6frvWD&6VE6W'fW'26F26W'fW"'&Ɨ7CffƖT'VvVUfFS2vWEfFT66TFW"vWEfFW26W'fW"'&Ɨ7CffƖT'VvVUfFS&VfVBWr'&Ɨ7Cₓb24VGBVBV&W$efFW226Rf"ffƖT'VvVUfFR66R2b66R4FVƗfW'7FFTF'GbbW'67E6W'fW%fFTFVƗfW'6W'fW"66R6FVSРb66R5&'&F67DFVBbb66RVVG4'&F67D6W'fW"6WC7G&sf'v&FVB6VE&'&F67B6V7F26vWF6W'fW"66RvWEWVB66RvWEW$R66RvWE6W'f6R66RvWEFR66RvWEFWBf6Rb66RvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVB66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRbW'67E6W'fW%fFTFVƗfW'6W'fW"66R6FVSРРР&VF6VBG'VSbvWD6frvWEvDf%W6W$ƖRb5W$ƖTf%fFU&WFr66RvWEW$RF6VBf6SV6Rb5W$ƖTf%fFU&WFr66RvWEW$RbbvWD7W'&VEW%6W'fW$f%fFU&WFr66RvWEW$RWV26W'fW"F6VBf6SРРbF6VB&V'&F67DW&R66RVVG4'&F67D6W'fW"b66R5&'&F67DFVBbb'&F67DW&PbbvWD6frvWE&'&F67DV&VB&VW$ƖR5W$ƖTf%fFU&WFr66RvWEW$R7G&rW%6W'fW"W$ƖRvWD7W'&VEW%6W'fW$f%fFU&WFr66RvWEW$RVð6WC7G&sF&vWG2&'&F67DFV6FW"&W6fUF&vWG2W$ƖRW%6W'fW"'&F67DW&R&'&F67DFV6FW"6VD'&F67B6W'fW"F&vWG2Рv&W76vU&FW"6VDW76vR6W'fW"FVfFuVvv&RfFR66RvWEW$R66RvWEWVB66RvWE6W'f6R66RvWEFRf6R66R5&VfFR66RvWEFWB66RvWEfFTBvWD6frvWD'VvVTvUFF2'&F67DW&RVV&W$efFW2FVVҲ&VfVBFB66RV6RFV'Vr$B6VFrfFR&V6W6RW6W"6wB6W'fW""6W'fW"# 66RF7G&rРРvWEfFT66TFW"&VfU6W'fW%fFW26W'fW"&VfVBV6RFV'Vr$66VBfFW2f"6W'fW#"6W'fW"РV6RFV'Vr$66VBfFW2f"6W'fW#"6W'fW"РРV6RFV'Vr%6W'fW"BfƖC"6W'fW"РРV&Ɩ276&旦VBfB6V6ƖUfFW27G&rW"7G&rWVB7G&r6W'fW"BFVb5W$ƖTf%fFU&WFrW"bbvWEfFT66TFW"4ƖUfFW2WVB'&Ɨ7CffƖT'VvVUfFS2vWEfFT66TFW"vWDƖUfFW2WVBb24VGb6W'fW"V’6W'fW"vWD7W'&VEW%6W'fW$f%fFU&WFrW"РbvWD6frvWD&6VE6W'fW'26F26W'fW"BVBV&W$efFW2B27G&V҂fFW"fFRfFR5&Wv&DFVƗfW&VB6VB&VFVƗfW&VE&Wv&Bf6S'&Ɨ7CffƖT'VvVUfFS&WFVBWr'&Ɨ7Cₓf"ffƖT'VvVUfFR66R2b66R5&'&F67DFVB6WC7G&sVFuF&vWG2WrƖVD66WC66RvWD'&F67EF&vWG2VFuF&vWG2&VfT†66RvWD'&F67Df'v&FVE6W'fW'2Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b&6VE6W'fW'2V’VFuF&vWG2&VfT†&6VE6W'fW'2Р66RvWD'&F67Df'v&FVE6W'fW'2FD‡6VE&'&F67BVFuF&vWG266RvWEWVB66RvWEW$R66RvWE6W'f6R66RvWEFR66RvWEFWBf6R66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRР&V'&F67DW&R66RVVG4'&F67D6W'fW"b66R5&'&F67DFVBbb'&F67DW&PbbvWD6frvWE&'&F67DV&VB7G&rW%6W'fW"6W'fW"V’6W'fW"vWD7W'&VEW%6W'fW$f%fFU&WFrW"6WC7G&sF&vWG2&'&F67DFV6FW"&W6fUF&vWG2G'VRW%6W'fW"'&F67DW&R&'&F67DFV6FW"6VD'&F67B6W'fW"F&vWG2Рb66R5&Wv&DFVƗfW&VBv&W76vU&FW"6VDW76vR6W'fW"FVfFuVvv&RfFTƖR66RvWEW$R66RvWEWVB66RvWE6W'f6R66RvWEFRf6R66R5&VfFR66RvWEFWB66RvWEfFTBvWD6frvWD'VvVTvUFF2'&F67DW&RVV&W$efFW2FR&VfVR26fƖB'&F67BFVƗfW'f"FP7W'&VBF&vWB&V6&BB6&WfW6ǒVFr7FFP&WG'6BV6RFR6RfFRvFW"ࠐb66R5&'&F67DFVBbb'&F67DW&R66RvWD'&F67Df'v&FVE6W'fW'2FB6W'fW"66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRР66R6WE&Wv&DFVƗfW&VBG'VRFVƗfW&VE&Wv&BG'VSFVVҲРb66R5&'&F67DFVBbb66R5&'&F67D6WFR&WFVBFB66RРРvWEfFT66TFW"&VfTƖUfFW2WVBf"ffƖT'VvVUfFRVFr&WFVBvWEfFT66TFW"FDƖUfFRWVBVFrРVF&VfVRǐbFVƗfW&VE&Wv&BbbvWD6frvWDVF&7W'BbbvWD6frvWDVF&Tv&&Wv&BVF&FW"6VD6V%fFRWVBW"РРРРР򢠠&WG&W2fFW"ֶWVB7FFR'&F67G2vVW"W2F&vW@&6VBf&R2VvW76vR6'&W"ࠒ&6W'fW"&6VB6W'fW"FBvVB6'&W &FV7FVB76&旦VBfB&WG'VFtƖT'&F67G27G&r6W'fW"Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b6W'fW"V&6VE6W'fW'2Vbb&6VE6W'fW'26F26W'fW"&WGW&㰠Рf"7G&r66VEWVBvWEfFT66TFW"vWDƖUfFUUTG2f"ffƖT'VvVUfFR66RWr'&Ɨ7CvWEfFT66TFW"vWDƖUfFW266VEWVBb66R4FVƗfW'7FFTF'GbbW'67DƖUfFTFVƗfW'66VEWVB66R6FVSРb66R5&'&F67DFVB66RVVG4'&F67D6W'fW"6FVSР6WC7G&sf'v&FVB6VE&'&F67B6V7F26vWF6W'fW"66RvWEWVB66RvWEW$R66RvWE6W'f6R66RvWEFR66RvWEFWBf6Rb66RvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVB66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRb66R5&Wv&DFVƗfW&VBbb66R5&'&F67D6WFRvWEfFT66TFW"&VfTƖUfFR66VEWVB66RV6RW'67DƖUfFTFVƗfW'66VEWVB66RРРРРР&FV7FVB76&旦VBfB&WG'VFuFT'&F67G27G&r6W'fW"Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b6W'fW"V&6VE6W'fW'2Vbb&6VE6W'fW'26F26W'fW"&WGW&㰠РbvWEfFT66TFW"vWEFT6vUVWVRV’&WGW&㰠Рf"fFUFUVWVRfFRWr'&Ɨ7CvWEfFT66TFW"vWEFT6vUVWVRbfFR4FVƗfW'7FFTF'GbbW'67EFUfFTFVƗfW'fFR6FVSРbfFR5&'&F67DFVBfFRvWEWVB4VGfFRvWD'&F67EF&vWG26F26W'fW"fFRvWD'&F67Df'v&FVE6W'fW'26F26W'fW"6FVSР6WC7G&sf'v&FVB6VE&'&F67B6V7F26vWF6W'fW"fFRvWEWVBfFRvWDRfFRvWE6W'f6RfFRvWEFRfFRvWEFF2f6RbfFRvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVBW'67EFUfFTFVƗfW'fFRРРР򢠠W&F6ǒ&WG&W2WfW'VFrfFW"ֶWVB7FFR'&F67BF20&WV&VBf"'&W"G&7'G2v6R&V6fW'FW2B&GV6RW"vࠒ6'&W"WfVBࠒV&Ɩ276&旦VBfB&WG'VFtƖT'&F67G2f"7G&r66VEWVBWrƖVD66WCvWEfFT66TFW"vWDƖUfFUUTG2f"ffƖT'VvVUfFR66RWr'&Ɨ7CvWEfFT66TFW"vWDƖUfFW266VEWVBb66R4FVƗfW'7FFTF'GbbW'67DƖUfFTFVƗfW'66VEWVB66R6FVSРb66R5&'&F67DFVB66R5&'&F67D6WFR6FVSР6WC7G&sVFuF&vWG2WrƖVD66WC66RvWD'&F67EF&vWG2VFuF&vWG2&VfT†66RvWD'&F67Df'v&FVE6W'fW'2Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b&6VE6W'fW'2V’VFuF&vWG2&VfT†&6VE6W'fW'2Р6WC7G&sf'v&FVB6VE&'&F67BVFuF&vWG266RvWEWVB66RvWEW$R66RvWE6W'f6R66RvWEFR66RvWEFWBf6Rb66RvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVB66R6WD'&F67Df'v&FVB66R5&'&F67D6WFRb66R5&Wv&DFVƗfW&VBbb66R5&'&F67D6WFRvWEfFT66TFW"&VfTƖUfFR66VEWVB66RV6RW'67DƖUfFTFVƗfW'66VEWVB66RРРРР&WG'VFuFT'&F67G2РV&Ɩ276&旦VBfB&WG'VFuFT'&F67G2bvWEfFT66TFW"vWEFT6vUVWVRV’&WGW&㰠Рf"fFUFUVWVRfFRWr'&Ɨ7CvWEfFT66TFW"vWEFT6vUVWVRbfFR4FVƗfW'7FFTF'GbbW'67EFUfFTFVƗfW'fFR6FVSРbfFR5&'&F67DFVBfFRvWEWVB4VG6FVSР6WC7G&sVFuF&vWG2WrƖVD66WCfFRvWD'&F67EF&vWG2VFuF&vWG2&VfT‡fFRvWD'&F67Df'v&FVE6W'fW'2Ɨ7C7G&s&6VE6W'fW'2vWD6frvWD&6VE6W'fW'2b&6VE6W'fW'2V’VFuF&vWG2&VfT†&6VE6W'fW'2Р6WC7G&sf'v&FVB6VE&'&F67BVFuF&vWG2fFRvWEWVBfFRvWDRfFRvWE6W'f6RfFRvWEFRfFRvWEFF2f6RbfFRvWD'&F67Df'v&FVE6W'fW'2FD†f'v&FVBW'67EFUfFTFVƗfW'fFRРРР&FV7FVB76&旦VB&VW'67EFUfFTFVƗfW'fFUFUVWVRfFRbvWEfFT66TFW"WFFUFUfFRfFRfFR6WDFVƗfW'7FFTF'Gf6R&WGW&G'VSРfFR6WDFVƗfW'7FFTF'GG'VR66VGVUFUfFTFVƗfW'&WG'&WGW&f6SР&fFRfB66VGVUFUfFTFVƗfW'&WG'bFUfFTFVƗfW'&WG'66VGVVBvWE66VGVW"V’&WGW&㰠РFUfFTFVƗfW'&WG'66VGVVBG'VSG'vWE66VGVW"66VGVR76&旦VBfFuVv&F2FUfFTFVƗfW'&WG'66VGVVBf6SР&WG'VFuFT'&F67G2RFUVB4T4E26F6'VFTW6WFRFUfFTFVƗfW'&WG'66VGVVBf6SFV'Vr%V&RF66VGVRFVB'&F67B7FFR&WG'"RvWDW76vRРР&FV7FVB76&旦VB&VW'67E6W'fW%fFTFVƗfW'7G&r6W'fW"ffƖT'VvVUfFRfFRbvWEfFT66TFW"WFFU6W'fW%fFR6W'fW"fFRfFR6WDFVƗfW'7FFTF'Gf6R&WGW&G'VSРfFR6WDFVƗfW'7FFTF'GG'VR66VGVT66VEfFTFVƗfW'&WG'&WGW&f6SР&FV7FVB76&旦VB&VW'67DƖUfFTFVƗfW'7G&rWVBffƖT'VvVUfFRfFRbvWEfFT66TFW"WFFTƖUfFRWVBfFRfFR6WDFVƗfW'7FFTF'Gf6R&WGW&G'VSРfFR6WDFVƗfW'7FFTF'GG'VR66VGVT66VEfFTFVƗfW'&WG'&WGW&f6SР&fFRfB66VGVT66VEfFTFVƗfW'&WG'b66VEfFTFVƗfW'&WG'66VGVVBvWE66VGVW"V’&WGW&㰠Р66VEfFTFVƗfW'&WG'66VGVVBG'VSG'vWE66VGVW"66VGVR76&旦VBfFuVv&F266VEfFTFVƗfW'&WG'66VGVVBf6SР&WG'66VEfFTFVƗfW'W'67FV6RRFUVB4T4E26F6'VFTW6WFR66VEfFTFVƗfW'&WG'66VGVVBf6SFV'Vr%V&RF66VGVR66VB'&F67B7FFR&WG'"RvWDW76vRРР&fFR76&旦VBfB&WG'66VEfFTFVƗfW'W'67FV6Rf"7G&r6W'fW"vWEfFT66TFW"vWD66VEfFW56W'fW'2f"ffƖT'VvVUfFRfFRWr'&Ɨ7CvWEfFT66TFW"vWEfFW26W'fW"bfFR4FVƗfW'7FFTF'GW'67E6W'fW%fFTFVƗfW'6W'fW"fFRРРРf"7G&rWVBWrƖVD66WCvWEfFT66TFW"vWDƖUfFUUTG2f"ffƖT'VvVUfFRfFRWr'&Ɨ7CvWEfFT66TFW"vWDƖUfFW2WVBbfFR4FVƗfW'7FFTF'GW'67DƖUfFTFVƗfW'WVBfFRРРРРV&Ɩ2fB6V6fFU'GbvWD6frvWEfFU'GV&VBbfFU'GfFW27W'&VEfFU'GfFW5&WV&VBFV'Vr%fFR'G&V6VB"FD7W'&VEfFU'GfFW27W'&VEfFU'GfFW5&WV&VB7W'&VEfFU'GfFW5&WV&VBvWD6frvWEfFU'G7&V6UfFW5&WV&VB6WEfFT66UfFU'G7&V6UfFW5&WV&VBvWEfFT66UfFU'G7&V6UfFW5&WV&VBvWD6frvWEfFU'G7&V6UfFW5&WV&VBbvWD6frvWEfFU'G'&F67B4VG'&F67BvWD6frvWEfFU'G'&F67BРf"7G&r6BvWD6frvWEfFU'G'VvVT6G2'V66T6B6BРbvWD6frvWEfFU'G6VEF6W'fW'2f"7G&r6W'fW"vWDf&U6W'fW'26VEfFU'G6W'fW"РV6Rf"7G&r6W'fW"vWD6frvWEfFU'G6W'fW'5F6VB6VEfFU'G6W'fW"РРР6fUfFT66TfRРРV&Ɩ2'7G&7BfBFV'Vr7G&r7G"&fFRfBFV'Vs"7G&rW76vRFV'VrW76vRР򢠠EE6ƖVBW6VBf"r&WVW7G2ࠒ&fFRfGG6ƖVBGG6ƖVBGG6ƖVBWt'VFW"6V7EFVWBGW&Fe6V6G2R'VB򢠠fWF6W2W"w2UTBg&FRrࠒ&W$RW"P&WGW&W"UTB"6FRVbBfV@F&w2W6WFbFR&WVW7Bf0F&w2FW''WFVDW6WFbFW''WFVBvRvFrf"FR&W76PV&Ɩ2UTBfWF6UTB7G&rW$RF&w2W6WFFW''WFVDW6WFbW$RVW$RWV4v&T66R&V"&WGW&VðРGG&WVW7B&WVW7BGG&WVW7BWt'VFW"W&U$7&VFR&GG3r6W6W'2&fW2֖V7&gB"W$RtUBFVWBGW&Fe6V6G2R'VBGG&W76S7G&s&W76RGG6ƖVB6VB&WVW7BGG&W76R&GFW'2e7G&rb&W76R7FGW46FRC&W76R7FGW46FRCBr%FW&R2W"vFFRR""W$R%""&WGW&VðРb&W76R7FGW46FR#&W76R7FGW46FR3F&rWrW6WF$fVBFfWF6UTBf""W$R"EE"&W76R7FGW46FRР6VVVBVVVB6'6W"'6U7G&r&W76R&GbVVVBVVVVB46&V7B&WGW&VðР6&V7B&V7BVVVBvWD46&V7Bb&V7B2&B"&V7BvWB&B"46V‚&WGW&VðР7G&rWVD57G&r&V7BvWB&B"vWD57G&r&WGW&'6UUTDg&7G&rWVD57G&rРV&Ɩ2'7G&7B6WC7G&svWDf&U6W'fW'26WFRFf&6W'fW"6WB&Vf&RvFVƗ7B&6VB&WFrfFW'2V&Ɩ2'7G&7B6WC7G&svWD6fwW&VE6W'fW'2V&Ɩ2'7G&7BfFuVv&6frvWD6frV&Ɩ2'7G&7B7G&rvWD7W'&VEW%6W'fW"7G&rW"򢠠&W6fW2W"w26W'fW"f"fFR&WFrFVF6FVBfFr&26W'26BW6W2FR&6VB&W6V6RG&6W"7FVBࠒ&FV7FVB7G&rvWD7W'&VEW%6W'fW$f%fFU&WFr7G&rW"b4FVF6FVEfFu&V&VB&WGW&&6VEW%&W6V6UG&6W"vWEW"W"&W6V6R&W6V6RvWE6W'fW"$V6RV“Р&WGW&vWD7W'&VEW%6W'fW"W"Р򢠠FVF6FVB&WFr2FVFǒVf&RVvW76vsF@G&7'B2GF6VBFW"f6r&BFW2B6''&6V@&W6V6R66G2ࠒ&FV7FVB&V4FVF6FVEfFu&V&VB&WGW&vWD6frvWDFVF6FVEfFu&bbWFBVbbWFB7W'G4&6VE&W6V6RРV&Ɩ2'7G&7BfRvWDFFfFW%VvₓV&Ɩ27G&rvWDFFF5vFFFUF6FFUFR5FRvWD'VvVUFT6V6W"vWEFR&WGW&vWDFFF5vFFFUF5FRРV&Ɩ27G&rvWDFFF5vFFFUF6FFUFR5FR&WGW&$FFF"5FRvWDFF7G&r""5FRvWEV"РV&Ɩ2'7G&7B7G&rvWE&W$R7G&rWVB7G&rW$RV&Ɩ2'7G&7B7G&rvWEUTB7G&rW$R&fFRBvWEfVR'&Ɨ7C6V627G&r6VBFFBf"6VB62bBvWDRWV4v&T66R6V⒒FFfVRfVRBvWEfVRBVbfVRV’&WGW&FFCРbfVR4BVfVRvWDBV6RbfVR57G&rG'VFVvW"'6TBfVRvWE7G&r6F6W6WFRv&PРР&WGW&VFFCРР&WGW&FFCР&fFRfFUFF566BvWE&V7FVE&fW%FF2'&Ɨ7C6VFF7G&rW"Ɨ7CFUGSFT6vW2vWDv&FFFW"vWEFT6vW2&V&W6WDFFT6vW26F2FUGRD&V&W6WEvVVFT6vW26F2FUGRtTT&V&W6WDFFT6vW26F2FUGRDB66WFVEVWVVEfFW2B66WFVDv&VWVVEfFW2f"fFUFUVWVRVWVVBvWEfFT66TFW"vWEFT6vUVWVRbVWVVB5&6W76VB66WFVDv&VWVVEfFW2РbVWVVB5&6W76VBbbVWVVBvWDRVbbVWVVBvWDRWV4v&T66RW"66WFVEVWVVEfFW2РРBfFT7&VVB66WFVEVWVVEfFW2BFUFFvWEfVRFF$FUFF"fFT7&VVBBFFF&W6WDFfFT7&VVBvWEfVRFF$FFF"fFT7&VVBBvVVǕFF&W6WEvVVfFT7&VVBvWEfVRFF%vVVǕFF"fFT7&VVBBFǕFF&W6WDFfFT7&VVBvWEfVRFF$FǕFF"fFT7&VVBBG2vWEfVRFF%G2"fFT7&VVBvWD6frvWEG4fFRBfFW2vWD6frvWDVDefFW5W$FbfFW2BF2vWD'VvVUFT6V6W"vWEFRvWDFdFbFFFF2fFW2FFFF2fFW3РРbvWD6frvWDƖ֗EfFUG2bbG2vWD6frvWDƖ֗EfFUG2G2vWD6frvWDƖ֗EfFUG2РBFFTFFFbvWD6frvWE7F&TFFF5vFFFRbvWD6frvWEW6TFFFUFF45&'FF‚FFTFFF&W6WDFfFT7&VV@vWEfVRFFvWDFFF5vFFFUFfFT7&VVBV6RFFTFFFFFFðРРE&V7FVEfFU'GvWE&V7FVEfFU'G7FFR66WFVDv&VWVVEfFW2&WGW&WrfFUFF566BFUFFFFFvVVǕFFFǕFFG2&V7FVEfFU'G&V7FVEfFU'GFFTFFF“Р&FV7FVB&V6f'v&E7FFT'&F67B&VvW5FF2&WGW&vW5FF3Р&FV7FVBEvWE&V7FVEfFU'G7FFRB66WFVEfFW2B7W'&VBfFU'GfFW3B&WV&VB7W'&VEfFU'GfFW5&WV&VCbvWD6frvWEfFU'GV&VB&WGW&WrE7W'&VB&WV&VBӰРB7&V6RvWD6frvWEfFU'G7&V6UfFW5&WV&VBf"B66WFVEfFW37W'&VBb7W'&VB&WV&VB7W'&VB&WV&VC&WV&VB7&V6SРР&WGW&WrE7W'&VB&WV&VBӰРV&Ɩ2'7G&7B7G&rvWEVvfW'6ₓV&Ɩ2'7G&7BBvWEfFT66T7W'&VEfFU'GfFW2V&Ɩ2'7G&7BrvWEfFT66T7EWFFVBV&Ɩ2'7G&7BBvWEfFT66U&WdFV&Ɩ2'7G&7B7G&rvWEfFT66U&WdFV&Ɩ2'7G&7BBvWEfFT66U&WevVVV&Ɩ2'7G&7BBvWEfFT66UfFU'G7&V6UfFW5&WV&VBV&Ɩ2'7G&7B&V5W$ƖR7G&rW$R򢠠6V62ƖR7FFRf"fFR&WFrW6r&6VB&W6V6RǒvVF0&2WƖ6Fǒ6fwW&VB2FRFVF6FVBfFr&ࠒ&FV7FVB&V5W$ƖTf%fFU&WFr7G&rW$R&WGW&4FVF6FVEfFu&V&VB&6VEW%&W6V6UG&6W"vWEW"W$R5&W6VB5W$ƖRW$RРV&Ɩ2'7G&7B&V56W'fW%fƖB7G&r6W'fW"V&Ɩ2'7G&7B&V56VTƖU6W'fW"7G&r6W'fW"&FV7FVB&V56VTƖU6W'fW$f%fFU&WFr7G&r6W'fW"b4FVF6FVEfFu&V&VB&WGW&56VTƖU6W'fW"6W'fW"Р6&V6FWfFwVv&&W6V6R&6VE&W6V6U7FGW27FGW2&6VEW%&W6V6UG&6W vWD&6VE7FGW26W'fW"&WGW&7FGW2Vbb7FGW24f&Rbb7FGW2vWEW$6VBРV&Ɩ2'7G&7B&V5fFT66Tv&UFRV&Ɩ2'7G&7Bח76frvWEfFT66Tו56frV&Ɩ2'7G&7Bח76frvWDfFVD66Tו56frV&Ɩ2'7G&7Bח76frvWEfFTvvtו56fr򢠠6WFFvו5&VFVB&W6W&6W26fVǒࠒV&Ɩ2fB6WFFvו7‚bvWE&ח7W76VvW"V’vWE&ח7W76VvW"6WFFvₓ6WE&ח7W76VvW"V“РbvWE&ו5‚V’vWE&ו5‚6WFFvₓ6WE&ו5†V“РРV&Ɩ2fBBfFT66R67F&vRfFVEW'57F&vRfFVD66T6WFB'VvVTWFBvWD'RvWD6frvWD'VvVTWFBbvWDWFBV’WFB'VvVTWFBTtU54tsРv&V7W'FVDFVF6FVEfFu&FRWVEW$T66RvWE&ו5‚vWE&w5UTDUVW''VvVUFT6V6W"6WEFT6vTf6fT'72vWD6frvWEFT6vTf6fT'72'VvVUFT6V6W"EFW"fFT66TFW"WrfFT66TFW"vWEfFT66Tו56frvWD6frvWEfFT66UW6Tו5‚vWD6frvWEfFT66UW6Tו5‚vWE&ו5‚vWDח7‚vWD6frvWDFV'Vr67F&vRfW'&FPV&Ɩ2fBtf7G&r6rtf6rРfW'&FPV&Ɩ2fBu6WfW&S7G&r6ru6WfW&R6rРfW'&FPV&Ɩ2fBFV'VsW6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vs7G&r6rbvWD6frvWDFV'VrFV'Vr6rРРfW'&FPV&Ɩ2fBFV'VsF&v&RRbvWD6frvWDFV'VrR&E7F6G&6RРРӰfFT66TFW"BfFVEW'466RWrfFVEW'466RvWDfFVD66Tו56frvWD6frvWDfFVD66UW6Tו5‚vWD6frvWDfFVD66UW6Tו5‚vWE&ו5‚vWDח7‚fFVD66T6vWD6frvWDFV'VrfW'&FPV&Ɩ2&VW6W$W7G27G&rWVB&WGW&vWE&ו5‚6F4WVW'WVBРfW'&FPV&Ɩ2fBtf7G&r6rtf6rРfW'&FPV&Ɩ2fBu6WfW&S7G&r6ru6WfW&R6rРfW'&FPV&Ɩ2fBFV'VsW6WFRbvWD6frvWDFV'VrR&E7F6G&6RРРfW'&FPV&Ɩ2fBFV'Vs7G&r6rbvWD6frvWDFV'VrFV'Vr6rРРfW'&FPV&Ɩ26WC7G&svWDUTG2&WGW&vWE&ו5‚vWEWVG2РӰbWFBWV2'VvVTWFBՕ5’G'&ח7W76VvW"Wrו7W76VvW"%fFuVv"vWE&ו5‚vWDח7‚vWD6V7FvW"vWDFF6W&6Rו7W76VvW"FR$V6W'fW$B$FP6rbvWD6frvWDFV'VrFV'Vr$vBg&"6r6W&6R#"6rVfVRvWE7V$6V‚" 6rVfVRvWDfVG2Рv&W76vU&FW"W76vR6rVfVRғ6F65W6WFRR&E7F6G&6RРV6RbWFBWV2'VvVTWFBTtU54trbvWD6frvWEVvW76vTV7'FₒV7'FFW"WrV7'FFW"%fFuVv"WrfRvWDFFfFW%Vvₒ'6V7&WFWW"РV6RbWFBWV2'VvVTWFB44UE2V7'FFW"WrV7'FFW"%fFuVv"WrfRvWDFFfFW%Vvₒ'6V7&WFWW"66WDFW"Wr66WDFW"vWEVvfW'6ₒvWD6frvWD'VvVT7BvWD6frvWD'VvVU'BV7'FFW"vWD6frvWDFV'VrfW'&FPV&Ɩ2fBr7G&r7G"tf7G"РӰ66WDFW"FBWr66WE&V6VfW"fW'&FPV&Ɩ2fB&V6VfTVfVR6VfVRVfVRv&W76vU&FW"W76vR6ۛh춻q^u͕!Yܤ($$%ɕɸ($%($%幍ɽ镐Aɕ͕!̤($$%չAAɕ͕!̡ܤ($$%Aɕ͕!̹СUե􁡅($$$%ɕՕ%ɕՕ%($$%($%(%((%ɥمєAɕ͕!YAAɕ͕!ܤ($%ɕɸձUեձѥ%ձ($$$$MՕ0($$$$܀􁡅ɕѕЀ܀ɕѕЀAIM9 }!9=}Q%5=UQ}5%11%L(%((%ɥمєMUU%ѕAAɕ͕!̡UU%ɕՕ%Mɥ͕ٕȰUU%%ɹѥ%($$%Mхѕаܤ($%1AAɕ͕!ѕ􁹕܁Ʌ1($%MUU%ѕÀ􁹕܁1!͡M($%幍ɽ镐Aɕ͕!̤($$%չAAɕ͕!̡ܤ($$%Aɕ͕!̹MРɕٕ%䀴($$$%AAɕ͕!􁕹九YՔ($$$%ɕՕ%Յ̡ɕՕ%($$$$%ɕɸ͔($$$%($$$%͕ٕȹՅ%ɕ ͔͕ٕȤ($$$$$$%ɹѥ%Յ̡%ɹѥ%($$$$$$MхѕЀ􁉅MхѕФ($$$$%ѕ($$$%($$$%ɕɸՔ($$%($%($%ȀAAɕ͕!聍ѕ($$%AAɕ͕ɕ͕􁉅AAɕ͕QɅȹAȡUե͔ձ($$%ɕ͕ձɕ͕MٕȠՅ%ɕ ͔͕ٕȤ($$$$$ɕ͕ ѥ%Յ̡ѥ%($$$%9ե͕ٕȤ($$$%ѕA̹Uե($$%($$%ɕ͕ѥѥ ($%($%ɕɸѕA(%(($($Ʌ́ٽѕȵ啐ɕ݅ɑ́ݡєɕٕ͹͡Ё($ɵ́ȁѕٽѥɽ丁 ɽ̵́ɔ($ɕɽ͕ѡȁѽչѥѠɔ፱ՑѼ($ٽ͕($(%ɽѕѕٽɽѕM͡1̡Mɥ͕ٕȰMUU%A̤($%ѕYѥAɽ͕ٕȀձ͕ٕȹ ($$%ɕɸ($%($%MUU%፱Ց􁡅Àձ ѥ̹MР聡A($%ȀAAɕ͕ɕ͕聉AAɕ͕QɅȹ=A̠($$%ɕ͕MٕȠՅ%ɕ ͔͕ٕȤ፱Ցх̡ɕ͕Uե($$$%ɕ͕A9ɕ͕UեѽMɥɕ͕MٕȠ($$%($%(%((%ɥمєٽ͍ɑAAɕ͕!Mɥե($%($$%UU%UեUU%ɽMɥեɥ($$%AAɕ͕!ɕٕ($$%幍ɽ镐Aɕ͕!̤($$$%ɕٕAɕ͕!̹ɕٔUե($$%($$%ɕ͕ѥѥ ɕٕ($%􁍅эፕѥɕ($$$%مѥѥ́ɔɕѕѡɕ͕Ʌȸ($%(%((%ɥمєٽ͍ɑAAɕ͕!AAɕ͕!($%ɕٕ􁙅͔($%幍ɽ镐Aɕ͕!̤($$%ձAɕ͕!̹СUե􁡅($$$%Aɕ͕!̹ɕٔUե($$$%ɕٕՔ($$%($%($%ɕٕ($$%ɕ͕ѥѥ ($%(%((%ɥمєٽ͍ɑAAɕ͕!̡Mɥ͕ٕȤ($%幍ɽ镐Aɕ͕!̤($$%Aɕ͕!̹MРɕٕ%䀴($$$%九YՔ͕ٕȹՅ%ɕ ͔͕ٕȤ($$$$%ɕɸ͔($$$%($$$%ɕ͕ѥѥ 九YՔ($$$%ɕɸՔ($$%($%(%((%ɥمєٽ͍ɑAAɕ͕!̡UU%ɕՕ%($%ɕՕ%ձ($$%ɕɸ($%($%幍ɽ镐Aɕ͕!̤($$%Aɕ͕!̹MРɕٕ%䀴($$$%ɕՕ%Յ̡九YՔɕՕ%($$$$%ɕɸ͔($$$%($$$%ɕ͕ѥѥ 九YՔ($$$%ɕɸՔ($$%($%(%((%ɥمєٽչAAɕ͕!̡ܤ($%Aɕ͕!̹MРɕٕ%䀴($$%AAɕ͕!􁕹九YՔ($$%܀􁡅ɕѕЀ܀ɕѕЀAIM9 }!9=}Q%5=UQ}5%11%L($$$%ɕɸ͔($$%($$%ɕ͕ѥѥ ($$%ɕɸՔ($%(%((%ɥمєٽɕ͕ѥѥ AAɕ͕!($%ձ($$%AAɕ͕QɅȹɕ͕ѥѥ Uե͕ٕȰ($$$$%MՕ($%(%((%ɽѕѕٽɕAAɕ͕!̡ܤ($%1AAɕ͕!ɕ􁹕܁Ʌ1($%幍ɽ镐Aɕ͕!̤($$%չAAɕ͕!̡ܤ($$%ȀAAɕ͕!Aɕ͕!̹مՕ̠($$$%UU%ѥٕIՕ%􁉅AAɕ͕QɅȹAM͡IՕ%͕ٕȰܤ($$$%ɕՕ%ձɕՕ%Յ̡ѥٕIՕ%($$$$%ɕՕ%ձ($$$%($$$%ɕՕ%ձ($$$$%ɕ久($$$%($$%($%($%ȀAAɕ͕!ɕ䤁($$%ɕՕ Aɕ͕M͡С͕ٕȰܰՔ($%(%((%ɽѕѕЁAAɕ͕! չР($%幍ɽ镐Aɕ͕!̤($$%ɕɸAɕ͕!̹ͥ锠($%(%((%ɽѕѕٽ͍ձ Aɕ͕MхI幌($%MձᕍѽM٥͍ձȀ􁝕MձȠ($%ѡձѡ Aɕ͕͍ձȀձ($$%ɕɸ($%($%͍ձȹ͍ձѡɕՕ Aɕ͕MхI幌($$$%AIM9 }MQIQUA}IMe9 }1e}M =9LQUйM =9L(%((%ɽѕѕٽɕՕ Aɕ͕MхI幌($%ѡձѡ Aɕ͕5ͅAɽ!Ȁձ($$%ɕɸ($%($%ɕՕѕЀMѕɕQ5̠($%Ё($%ȀMɥ͕ٕȀ聝مMٕ̠($$%Aɕ͕MٕY͕ٕȰYѥA՝]ɔMU }AIM9 }IMe9 }IEUMP($$$%ѥՔ($$%($$%5ͅAɽ!ȹ͕5͕ٕͅȰ䬬($$$$%YѥA՝]ɔɕ͕I幍IՕС͕ٕȰUU%ɅUU%ɕՕѕФ($%(%((%ɥمєٽх Aɕ͕($%ѡձѡ Aɕ͕($$%ɕɸ($%($%ɕ Aɕ͕AIM9 }  -9}Q%5=UQ}5%11%L($%ȀMɥ͕ٕȀ聝مMٕ̠($$% IٕM̹͡х̡ɕ͕Mٕ-͕ٕȤ($$$%ɕՕ Aɕ͕M͡С͕ٕȤ($$%($%($%ɕAAɕ͕!̡MѕɕQ5̠(%((%ɥمєMɥɕ͕Mٕ-Mɥ͕ٕȤ($%ɕɸ͕ٕȀձ͕ٕȹɥѽ1ݕ ͔لѥ1I==P(%((%ɥمєAɕ͕MٕYMɥ͕ٕȰMɥՉ ($$Qɕ͕ɽѽ́Ёչ䁥́ѡɕ͕иQ($$͕ѕɅЁЁ䁉ͥѼ͕ٕ́ѕЁѼ($$ͽєѡȸ($%͕ٕȀձ͕ٕȹ MٕY͕ٕȤ($$%՜%ɕՉ ɕ͕ٕȁչɕ͕ٕȈ($$%ɕɸ͔($%($%ɕɸՔ(%((%ɥمєAɕ͕ɅѥYUU%%ɹѥ%Mхѕа($$%ɕ͕QхMɥՉ ($%%ɹѥ%ձMхѕЀ0ɕ͕QхMхѕФ($$%՜%ɕՉ ɕ͕ٕݥѠمɅѥ($$%ɕɸ͔($%($%ɕɸՔ(%(($($Íٕɕ͕ݹ䁉́ѡЁٔѽɕѥщ̸($MձѥЁɅѥɔѕѥ䁱ЁѼѕ($ɽ䁵($($Ʌѥ5́᥵մͥɔ($ɕɸɕ͕ٕȁ($(%ՉMMɥɕ Aɕ͕ѥ5̤($%ѡձѡ Aɕ͕($$%ɕɸ ѥ̹MР($%($%܀MѕɕQ5̠($%MMɥɕ􁉅AAɕ͕QɅȹɕ ̡ܰѥ5̤($%ȀMɥ͕ٕȀ聕ɕ($$%͍ɑAAɕ͕!̡͕ٕȤ($$$-ɕٕݡѡ́Ʌѥ́չم%ѡͅ($$$ɽ́ɕյ̰́щЁɬЁمѡ($$$ѕхͬݥɕՕЁɕ͠͹͡Ё́ݡх啐($$% IٕM̹͡ɕ͕Mٕ-͕ٕȤ($%($%幍ɽ镐Aɕ͕!̤($$%չAAɕ͕!̡ܤ($%($%ɕɸɕ(%((%ՉٽMɥ9MɥեMɥ͕ٕ9($% =5($$%ե􁝕UU%9($%(($%($$%եձե䠤եՅ%ɕ ͔ձ($$$%եUU%ɽMɥեɥѽMɥ($$%($%􁍅эፕѥɕ($$$ɔ($%(($% =5($$%9YѕAȡե9($%($%A=YѕIѥ9($$% х($$$%х!ȠQ !($$$$%х!Ƞ͡Q ̠($$$%($$%(($$% Yѕ̡͕ٕ9($$%ɕA= ɽ̡͕ٕ9($$%ɕAQ ɽ̡͕ٕ9($$%=Yѕ̡9ե͕ٕ9($$%ձѥAɽ!ȹե9($%(%((%ɥمєٽ%Mɥ͜($%͜(%((%ՉɅЁٽMٕɔMɥͅ((%Չٽͅ($%ѽ ɽM٥̡͔($%Yѕ !ȠٕͅYѕ (($%Aɽ5ű5͕Ƞձ($$%Aɽ5ű5͕Ƞ͡ёݸ($%(($%Aɽ5ME0ձ($$%Aɽ5ME0͡ёݸ($%($%ձѥAɽ!Ȁձ($$%ձѥAɽ!ȹ͔($%(($%ͽ!Ȁձ($$%ͽ!ȹ͕ ѥ($%(($%ɕ!Ȁձ($$%ɕ!ȹ͔($%($%ɕAՉ͡Aձ($$%ɕAՉ͡A͔($$%ɕAՉ͡Aձ($%(($%չQ ȹ͡ёݸ(($%х!Ƞձ($$%х!Ƞ͡ёݸ($%(($%􁙅͔(%((%ՉٽA՝5ͅIٕх%Mɕ($%չ幌($$%($$$%MɥMՈ($$$% A՝5ͅѥѥ!Ȁձ($$$$%MՈ􁕹ѥ!ȹСɕUQ($$$%􁕱͔($$$$%MՈ􁥸ɕUQ($$$%(($$$%Ёͥ销􁥸ɕ%Р쀼ͅ䁽(($$$% ՜($$$$%՜Iٕ՝ͅ􈀬MՈͥ􈀬ͥ锤($$$%(($$$%Mɥ屽􀈈($$$%ͥ销($$$$% A՝5ͅѥѥ!Ȁձ($$$$$%屽􁕹ѥ!ȹСɕUQ($$$$%􁕱͔($$$$$%屽􁥸ɕUQ($$$$%($$$%(($$$%)ͽٕٕ)ͽٕ 屽(($$$%MՈՅ%ɕ ͔ٕMՉ ($$$$% ՜($$$$$%݅ɸA՝5ͅՉ ͵э聡􈀬MՈ($$$$$$$$ٕMՉ ($$$$%($$$$%ɕɸ($$$%(($$$%5ͅAɽ!ȹ5ٕͅ($$%􁍅эፕѥ($$$%ɥMхQɅ($$%($%(%((%ɥمєUU%͕UU%ɽMɥMɥեMɥ($%Mɥmt̀쀈եMɥՉɥरեMɥՉɥఀȤ($$$$եMɥՉɥȰؤեMɥՉɥذ($$$$եMɥՉɥȤ(($%M ̀1ltYՔ($%M ̀($%M ́1ltYՔ($%M ̀($%M ́1ltYՔ(($%M ̀1ltYՔ($%M ̀($%M ́1ltYՔ(($%ɕɸ܁UU%M ̰M ̤(%((%Չ幍ɽ镐ٽɽEՕՔ($%ݡYѕ !ȠQ EՕՔͥ锠($$%YѕQEՕՔٽє􁝕Yѕ !ȠQ EՕՔР($$%ٽєAɽ͕($$$%YѕQхM͡ЁՕՕQх̀ٽєQх̠ձٽєQх̠䠤ձ($$$$$$YѕQхM͡й͕MѽɅٽєQх̠($$$%EՕՕYѕIձЁɕձЀٽєٽє9ٽєM٥Ք͔ٽєQՕՕQх̰($$$$$%ٽєUեٽє($$$%ɕձЀEՕՕYѕIձйIQId($$$$%͍ձQYѕI䠤($$$$%ɕɸ($$$%($$$%ɕձЀEՕՕYѕIձйQI5%90($$$$%݅ɸI٥ѕɵɽٕȁٽєٽєYѕ%Ȁٽє9($$$$$$$M٥MѕYѽȹͅѥ1ٽєM٥($$$%($$%($$%Yѕ !ȠɕٕQYєٽє($$$%͍ձQYѕI䠤($$$%ɕɸ($$%($%(%((%ɥمєٽ͍ձQYѕI䠤($%ѥYѕIMձMձȠձ($$%ɕɸ($%($%ѥYѕIMձՔ($%($$%MձȠ͍ձ($$$%幍ɽ镐YѥA՝Aɽѡ̤($$$$%ѥYѕIMձ􁙅͔($$$%($$$%ɽEՕՔ($$%԰QUйM =9L($%􁍅эIչѥፕѥ($$%ѥYѕIMձ􁙅͔($$%՜UѼ͍ձɽٕȁٽєɕ耈5ͅ($%(%((%Չٽɕ($%ɕIչѥՔ(%(($́ ɽɥѕɅѥɕݥѡЁѽ́ѽȁȁѕ͕٥(%Չٽɕɽ ɽ($%ɕIչѥ͔(%((%ɥمєٽɕIչѥɕх ɽM٥̤($%ѡ չ5ѡ 9 չ5ѡ($%5ѡձ($$%ѡ չ5ѡA1U%95MM%9($%($%݅ɹUѕѕYѥAɽ5(($%͕ ɕYѕAYѕIեɕ($$$% YѕAYѕIեɕYѕ YѕA%ɕ͕YѕIեɕ($%5ձѥAɽMР($%ɕх ɽM٥̤($$%х ɽM٥̠($%(%((%ɥمєٽ݅ɹUѕѕYѥAɽ5($% ѕYѥAɽ䠤ѡձѡ Aɕ͕($$%MٕɔѕYѥAɽɕեɕ́5eME0I%L5EQPȁM= -QLA1U%95MM%9́ͅȀ($$$$$ѕɽɽѥѼɵɽɽѥ($%(%((%ՉɅЁٽչ幌Iչո(($Aљɴ͕䁙ȁѡɅеɅ ɽ͍ٕ䁍Ʌи(%ՉɅЁMɥAɽAљɴ((%ՉɅЁٽչ ͽ Mɥ((%ՉɅЁٽٕͅYѕ ((%ՉɅЁٽɕ ɔŰ(($MɥЁ ɽɕѠ쁙ɕ́ɽєͼѡȁɕѽɔ́(%ՉɅЁٽɕ ɽ Ʌѥѡɽ́ፕѥ((%ՉɅЁ͕A՝5ͅфMɥ͕ٕȰMɥѕmtфՕՔ((%ɥمєхѥЁA1U%9}5MM}!I}1%5%P(%ɥمєхѥЁA1U%9}5MM}M=Q}1%5%P((%Չٽ͕A՝5ͅMٕȡMɥ͕ٕȰЁ䰁)ͽٕٕ($%MձȠ͍ձ͕A՝5ͅMٕ9ܡ͕ٕȰٕ䀨0QUй5%11%M =9L(%(($($Ḿ՝ٕͅѕ䁅ɕ́ݡѡȁѡɽ($ѕЁȁٕ($($Ʌ͕ٕȁхɝЁ͕ٕ($ɅٕٕѼ͕($ɕɸՔݡѡɽ䁅ѕѡͅȁٕ($(%ɽѕѕ͕A՝5ͅMٕ9ܡMɥ͕ٕȰ)ͽٕٕ($%MɥՉ 􁕹ٕMՉ ($%Mɥ屽)ͽٕ ٕ(($%ѕmtՉ ѕ̀Չ ѕ̡ل͕йMхɑ ͕̹UQ|($%ѕmt屽 ѕ̀屽 ѕ̡ل͕йMхɑ ͕̹UQ|(($$ѥєѕ́ɥѕ($$ɥѕUQ̀ȵєѠɕUQѕ($$ɥѕ%Ё̀Ёѕ($%ЁѥѕM销ȀՉ ѕ̹ѠՉ UQɕѕ̤($$$$Ѐ屽Ѡ($$$$Ȁ屽 ѕ̹Ѡ쀼屽UQɕѕ̤(($%ѥѕM销A1U%9}5MM}M=Q}1%5%P($$%՜mA՝5ͅtA屽ɥЀѥѕM销ѕ̤͕ٕ􈀬͕ٕ($$$$$Չ 􈀬Չ PͥȁÍѕ($%(($%ѥѕM销A1U%9}5MM}!I}1%5%P($$%՜mA՝5ͅtA屽Q=<1IѥѕM销ѕ̰􈀬A1U%9}5MM}!I}1%5%P($$$$$͕ٕ􈀬͕ٕȀՉ 􈀬Չ P9=P͕Ј($$%ɕɸ͔($%(($%䀡 ѕɅ=Mɕѕ=Mɕ􁹕܁ ѕɅ=Mɕ($$$%х=MɕЀ􁹕܁х=Mɕѕ=Mɕ($$% A՝5ͅѥѥ!Ȁձ($$$%йɥѕUQѥ!ȹСՉ ($$%􁕱͔($$$%йɥѕUQՉ ($$%(($$$ͅ䁽5UMPѕ̰Ё($$%йɥѕ%С屽 ѕ̹Ѡ(($$% A՝5ͅѥѥ!Ȁձ($$$%йɥѕUQѥ!ȹС屽($$%􁕱͔($$$%йɥѕUQ屽($$%($$%й͠(($$%͕Ѐ͕A՝5ͅф͕ٕȰ A՝5ͅ ѽ1ݕ ͔($$$$%ѕ=Mɕѽ ѕɅ䠤͔($$% ՜($$$%՜͕ЀMЈ耉 ձЁ͕՝ٕѥѕM销ѕ̤͕ٕ($$$$$$Չ ٕ̠($$%($$%ɕɸ͕($%􁍅эፕѥ($$%ɥMхQɅ($$%ɕɸ͔($%(%((%ɥمєձ) եI ($%ձ) եȁձ) եȠ($$$$х͔ I%ࠤѥQ5̠ͽQ5̠($% IU͕ɹձ IU͕ɹ䠤($$%͕ȡ IU͕ɹ($%($% IAݽɐձ IAݽɐ䠤($$%ݽɐ IAݽɐ($%($%ɕɸե(%((%Չ͕IٕMٕȡMɥ͕ٕȰ)ͽٕٕ($%ɕɸ͕IٕMٕȡ͕ٕȰٕ͔(%((%ɥمє͕IٕMٕȡMɥ͕ٕȰ)ͽٕٕ͕I ݸ($%)AՉ͡AɕAՉ͡A($%Չ͡Aձ͕I ݸMѕɕQ5̠ɕAՉ͡IѕȤ($$%ɕɸ͔($%(($%䀡)́̀Չ͡AIͽɍ($$%Mɥ􁝕 IAɕࠤYѥA՝|͕ٕ($$%Չ͍ɥ̀􁩕̹Չ͠)ͽٕ ٕ($$%ɕAՉ͡IѕȀ0($$%ɕɸՉ͍ɥ̀($%􁍅эፕѥ($$%͕I ݸ($$$$Mхɽ́ɕՕՕͼѡȁɕɥ́ѡɽѱͅ($$$%ɕAՉ͡IѕȀMѕɕQ5̠0($$%($$%՜5ͅ($$%ɕɸ͔($%(%((%Չ͕5ٕMٕȡMɥ͕ٕȰ)ͽٕٕ($%!Ȁձ($$%ɕɸ͔($%($%($$%!ȹՉٕ͡ 5Aɕࠤٽѥ՝͕ٕ̼͕ٕȰٕ($$%ɕɸՔ($%􁍅эፕѥ($$% ՜($$$%ɥMхQɅ($$%($$%ɕɸ͔($%(%((%Չ͕MٕMٕȡMɥ͕ٕȰ)ͽٕٕ($%5Mɥ=Ʌѥ􁝕 MMٕ Ʌѥ͕ٕȤ($%Ʌѥձ($$%ɕɸ͔($%($%MɥЀ􁍽ɅѥР!ЈхMɥMɥɅѥР!Ј耈($%ЁЀ􁍽ɅѥРAЈх9յȀ9յȤɅѥРAЈYՔ($%й䠤($$%ɕɸ͔($%(($%Mɥ屽)ͽٕ ٕ($%Mɥ􁕹ѥ!Ȁձѥ!ȹС屽屽($%䀡MЁͽЀ􁹕܁MР($$%ͽйС܁%Mɕ̡аФ($$%䀡х=MɕЀ􁹕܁х=Mɕͽй=Mɕ($$$%йɥѕUQ($$$%й͠($$%($$%ɕɸՔ($%􁍅эፕѥ($$%՜5ͅ($$%ɕɸ͔($%(%((%Չٽ͕Mٕ95ͅ($%ȀMɥ̀聝مMٕ̠($$%͕A՝5ͅMٕȡ̰İYѥA՝]ɔ͕ٕ9̤($%(%((%Չٽ͕YѕAMɥ͕ٕȤ($%M=MٕYѕIѥ͕ٕȤ($$%5ͅAɽ!ȹ͕5͕ٕͅȰİYѥA՝]ɔٽѕA չ($%(%((%Չٽ͕ ɕYѕAYѕ̡ЁչФ($%ٽѕAYѕ̀􁅵չ($%͕Yѕ YѕA ɕYѕ̡չФ($%՜ ɕЁٽєѽх耈ٽѕAYѕ̤(%((%ՉɅЁٽ͕Yѕ 1Uѕ((%ՉɅЁٽ͕Yѕ Aɕ䡥Ё((%ՉɅЁٽ͕Yѕ Aɕ5ѠMɥѕФ((%ՉɅЁٽ͕Yѕ Aɕ]Ёݕ((%ՉɅЁٽ͕Yѕ Yѕ %ɕQɔ((%ՉɅЁٽ͕Yѕ YѕA ɕYѕ̡Ёٽѕ̤((%ՉɅЁٽ͕Yѕ YѕA%ɕ͕YѕIեɕЁٽѕ̤((%Չٽх̠($%ȀMɥ̀聝مMٕ̠($$%M=MٕYѕIѥ̤($$$%9͕ٕ́Ȁ̀Ѽ͕ѕЁх͔́ͅɕѕЁݥѠͽ($$%􁕱͔($$$%MɕՕЁȁх̤́ͅ($$$%5ͅAɽ!ȹ͕5̰ͅİYѥA՝]ɔх̡̤($$%($%(%((%ɥمєٽ͕YѕIѕMɥȰMɥեMɥ͕٥=($$%MɥMٕȤ($%=MٕȀձمMٕ̠х̡MٕȤ($$%՜9Ё͕ٽєɕѥȀȀ͔ѡȁ́($$%ɕɸ($%(($%5ͅAɽ!ȹ͕5ͅMٕȰİ($$$%YѥA՝]ɔٽѕIѕȰե͕٥Ք(%((%ՉMɥ]UѥMѕɽM٥Mɥ͕٥($%ȀMɥͥє聝 ]UѥYѕMѕ̠($$% ]UѥYѕM٥ͥєՅ%ɕ ͔͕٥($$$%ɕɸͥє($$%($%($%ɕɸ(%((%ɥمє1YѕQMɥեɅ1 յ̰MɥͥєMɥ͕٥MɥȰ($$%ՑQ EՕՔ($%IQ(($%Yѕ !Ƞ=Yѕ̡ե($$%Ʌ1= չYєYѕ̀􁝕Yѕ !Ƞ=Yѕ̡ե($$%Ȁ= չYєٽє聽Yѕ̤($$$%ٽєM٥Յ%ɕ ͔͕٥($$$$%IQ5ѠࡵIQٽєQ($$$%($$%($%(($%ȀMɥ͕ٕȀ聝مMٕ̠($$%Ȁ= չYєٽє聝Yѕ !ȠYѕ̡͕ٕȤ($$$%ٽєUեՅ̡եٽєM٥Յ%ɕ ͔͕٥($$$$%IQ5ѠࡵIQٽєQ($$$%($$%($%(($%ՑQ EՕՔȀձ($$%ȀYѕQEՕՔՕՕYє聝Yѕ !ȠQ EՕՔ($$$%ՕՕYє9Յ%ɕ ͔Ȥ($$$$$$ՕՕYєM٥Յ%ɕ ͔͕٥($$$$%IQ5ѠࡵIQՕՕYєQ($$$%($$%($%(($%Ȁ յ聍̤($$%9Յ%ɕ ͔1Yѕ̈($$$%хYՔمՔ􁐹YՔ($$$%MɥmtЀمՔMɥР($$$%ȀMɥȀ聱Ф($$$$%MɥmtфȹР($$$$%хltՅ%ɕ ͔ͥє($$$$$%IQ5ѠࡵIQ1مՕ=хlt($$$$%($$$%($$%($%($%ɕɸIQ(%((%ՉYѕMɥեMɥ͕٥Ʌ1 յф($%ɕɸYѕեձ͕٥ф͔(%(($($ ́ѡɕٽє䰁ѥ䁥Ցѕٽѕ́݅ѥ($ȁфѥѼ͠($($ɅեȁUU%($Ʌȁȁ͕ѡѥՕՔ($Ʌ͕٥ٽє͕٥($ɅфɕЁȁф($ɅՑQ EՕՔݡѡȁՕՕٽѕ́ɕ͕ٔѡȁͱ($ɕɸՔݡѡٽє䁉ѕ($(%ՉYѕMɥեMɥȰMɥ͕٥Ʌ1 յф($$%ՑQ EՕՔ($%Mɥͥє􁝕]UѥMѕɽM٥͕٥($%ͥє䠤($$%՜9͕٥ͥє͕ЁȀ͕٥ͭٽє䁍($$%ɕɸՔ($%(($%Ёٽѕ􁝕 ]UѥYѕYѕͥє($%Ёٽѕ5􁝕 ]UѥYѕYѕ5ͥє(($%Yє􁝕1YѕQեфͥє͕٥ȰՑQ EՕՔ($%Yє($$%՜9ЁٽєѥչȀե͕٥ͭٽє䁍($$%ɕɸՔ($%(($%($$%1ѕQ܀􁝕 չQ ȠQ($$%1ѕQYѕQ1ѕQ%хС%хй5Yєi%ѕձР($$$$$!̡ Q!=MР(($$% ]UѥYѕYѕͥє($$$%ٽѕٽѕ5($$$$%՜Yє䁥̀Ȁͥєͭٽє䁍($$$$%ɕɸՔ($$$%(($$$%1ѕQٽє􁱅YѕQ!̠ٽѕ䤹5ѕ̠ٽѕ5($$$%ɕɸܹѕȡٽє($$%($$%1ѕQɕ͕Q􁱅YѕQݥѡ!ȡ ]UѥYѕYѕ!ȡͥє($$$$$ݥѡ5єݥѡM($$%1ѕQɕ͕QQɽ܀ɕ͕Q!̠Ф(($$%YѕQ ɔɕ͕Q($$$%ܹѕȡɕ͕Q($$$$%՜Yє䁥́ЁȀե͕٥ٽєɽ͕($$$$%ɕɸՔ($$$%($$%􁕱͔($$$%ܹѕȡɕ͕QQɽܤ($$$$%՜Yє䁥́ЁȀե͕٥ٽєɽ͕($$$$%ɕɸՔ($$$%($$%($%􁍅эፕѥ($$%ɥMхQɅ($%(($%՜Yє䁥́ЁЁȀե͕٥ͭٽє($%ɕɸ͔(%((%Չ幍ɽ镐ٽٽєMɥȰMɥ͕٥ɕYєѥEՕՔՕՕQ($$%YѕQхM͡ЁѕаMɥե($%ٽєȰ͕٥ɕYєѥEՕՔՕՕQѕаեձ(%((%ɥمєմEՕՕYѕIձЁ($%MU MLIQIdQI5%90(%((%ɥمє幍ɽ镐EՕՕYѕIձЁٽєMɥȰMɥ͕٥ɕYєѥEՕՔՕՕQ($$%YѕQхM͡ЁѕаMɥեYѕQEՕՔՕՕYє($%($$%M٥MѕYѽȹY͕٥($$$%݅ɸIѕٽєݥѠم͕٥ͥєM٥MѕYѽȹͅѥ1͕٥($$$%ɕɸEՕՕYѕIձйQI5%90($$%($$%5ɅU͕ɹYѽȹYȰ ɽAAɕࠤ($$$%݅ɸIѕٽєݥѠم5ɅЁ͕ɹ($$$$$$5ɅU͕ɹYѽȹͅѥ1Ȥɽ͕٥($$$$$$5ɅU͕ɹYѽȹͅѥ1͕٥($$$%ɕɸEՕՕYѕIձйQI5%90($$%(($$%UU%ٽѕ%ՕՕYєձձՕՕYєYѕ%($$%ٽѕ%ձ($$$%ٽѕ%UU%ɅUU%($$%(($$$UU%ɕͽѥ($$% =5($$$%ե􁝕UU%Ȥ($$%(($$%եձե䠤($$$%ե􁝕UU%Ȥ(($$$$ ɽɕѼѕ($$$%ե䠤 ɽAAɕࠤ䠤($$$$$$ȹх]Ѡ ɽAAɕࠤ($$$$%MɥեĀ􁝕UU% ɽAAɕࠤȤ($$$$%եĹ䠤($$$$$%՜ѕѕɽȁݥѡЁɕఁѥ($$$$$%Ȁ􁝕 ɽAAɕࠤ($$$$$%եե($$$$%($$$%($$%(($$%ե䠤($$$%ȹх]Ѡ ɽAAɕࠤ($$$$%%ɥٽєͥչѼЁUU%ɽȈ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%($$$% U)($$$$%%ɥٽєɽȀͥȁ͸Ёɔ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%($$$% UU%1($$$$%ѼЁեȀȤ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%(($$$%՜эUU%ͥչ́($$$%UU%Ԁձ($$$%($$$$% =5($$$$$%Ԁ􁙕эUU%Ȥ($$$$%($$$%􁍅эፕѥ($$$$% ՜($$$$$%ɥMхQɅ($$$$%($$$%($$$%Ԁձ($$$$%՜ѼЁեȀȤ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%($$$%եԹѽMɥ($$%(($$$9ɵ锁UU%ɥͥ($$%($$$%եձե䠤եՅ%ɕ ͔ձ($$$$%եUU%ɽMɥեɥѽMɥ($$$%($$%􁍅эፕѥɕ($$$$ɔ($$%(($$%Ȁ􁝕Aɽ9եȤ(($$$ хє͕ٕȁ%5A=IQ9PȁɽЁɕѹ̤($$%=A=YѕIѥȤ($$%MɥMٕȀ= ɕAMٕYѕIѥȤ聹ձ($$%ѥՕՕQՕՕQ($$$$$1ѕQܠii%ѕձРѽ%хРѽ5(($$%MMɥɽQɝ̀ՕՕYєձ܁1!͡M($$$$$聹܁1!͡MՕՕYє ɽQɝ̠($$%MMɥɽ݅ɑMٕ̀ՕՕYєձ܁1!͡M($$$$$聹܁1!͡MՕՕYє ɽ݅ɑMٕ̠($$%ɽ ɽ!ՕՕYєձՕՕYєAɽ ɽ!($$%ɽ͕Qх̀􁝕 AɥMٕȠ 5ձѥAɽMР($$%Qх̀ɽ͕Qх̀ չ5Qх̠($$%YѕMх ɽЀ􁍅݅ɑMх ɽСQх̤($$%Ʌ1 յфձ($$%ՕՕQ 􁙅͔(($$$ѥݥѽх́ɕ䁽ȁՕՕٽѕ̸Iո($$$ɔѡ́ٽє́х͔͹͡Ёͼѡձѥ́܁͔($$$ѡеɽٕȁхє($$% хх!ȠQ !($$$%х!Ƞ͡Q ̠($$$%ՕՕQ ѥEՕՔх!ȠQ !($$%(($$$Yєѡٽєɔ䁥єչиQ́́є($$$ٽѕ́ɕѕѡ䁍ЁѡфɽٕȁՕՔ($$$ɕٕ́չٽєѡЁݥЁɽ͕($$%Qх̤($$$%Aɽ5ME0ձ($$$$%Mٕɔ5Ű́Ёɕѱ䰁ѽٽєɽͥ($$$$%ɕɸEՕՕYѕIձйIQId($$$%(($$$%Aɽ5ME0х-EՕե($$$$%Aɽ5ME0єեA9܁хYՕMɥȤ($$$$%Aɽ5ME0Uե̠ե($$$%(($$$%ф􁝕Aɽ5ME0ᅍEՕ䡹܁ յե܁хYՕMɥե($$$%YѕեȰ͕٥фՕՕYєձ($$$$%Yє䁥́ЁЁȀȀ͕٥ͭٽє($$$$%͕YѕIѕȰե͕٥=MٕȤ($$$$%ɕɸEՕՕYѕIձйQI5%90($$$%($$%(($$$݅ɐѕɽЁɔѡѥѥٔф($$$ՕՕ́ѡɕ݅ɐѽх́ݽɬQՕՕٕхєɕٕ($$$ɕ她ɽ́ѡЁɕɕ($$%ՕՕQ ($$$%YѕQхM͡ЁɽѕQх̀􁵅Qх̀AɽѕIٕQх̡фȤѕ($$$%YѕMх ɽЀɽ ɽȹ͕%ѕ݅ɑ=($$$$%ɽQɝ̹ɽ ɽȹɕͽٕQɝ̡͔ձ($$$$%ɽ ɽ!Ք($$$%($$$%YѕQEՕՔ啑Yє􁹕܁YѕQEՕՔٽѕ%Ȱ͕٥ѥ($$$$$%ɽ ɽ!ɽQɝ̰ɽ݅ɑMٕ̰($$$$$%ɽѕQх̀ձɽѕQх̹ѽMɥ͔ե($$$%Yѕ !ȠQYѕQ 啑Yє($$$$%MٕɔUѼͥЁՕՕɽٕȁٽєȀȀ͕٥($$$$$$$ͭɽ䁉ɽЈ($$$$%ɕɸEՕՕYѕIձйIQId($$$%($$$%ɽ ɽ!($$$$%ȀMɥхɝЀ聉ɽQɝ̤($$$$$%MMɥ݅ɑ͕Aɽ ɽС ѥ̹ͥѽхɝФեȰ($$$$$$$%͕٥ѥɽѕQх̀ձɽѕQх̹ѽMɥ͔($$$$$%啑Yє ɽ݅ɑMٕ̠݅ɑ($$$$$$%ɽ݅ɑMٕ̹݅ɑ($$$$$$%ͥQYѕٕ䡑啑Yє($$$$$%($$$$%($$$%($$$% ٽєɽȀ͕٥($$$$$$͔ѥ́ɥЁ܈($$$%ɕɸEՕՕYѕIձйMU ML($$%(($$%YѕA䠤(($$$Qх́ɽͥɥ͕ٕȁ=Hձѥɽ($$%ɽ͕Qх̤($$$%Qх̤($$$$%ЁQQх􁝕YՔфQQхĤ($$$$%ЁѡQх􁝕YՔф5ѡQхĤ(($$$$%Ёѕ5ѡQх($$$$% Mѽɕ5ѡQх]ѡє($$$$$% U͕5ѡѕQхAɥQх($$$$$$%ѕ5ѡQх􁝕YՔф5ѡQх]ѡѕAѠĤ($$$$$%􁕱͔($$$$$$%ѕ5ѡQх􁵽ѡQх($$$$$%($$$$%(($$$$%ЁݕQх􁝕YՔф]QхĤ($$$$%ЁQх􁝕YՔфQхĤ($$$$%Ё̀􁝕YՔфÄ A=Yє(($$$$%ЁYѕ̀􁝕 5չ=YѕA䠤($$$$%Yѕ̀($$$$$%1ѕQQ􁝕 չQ ȠQ($$$$$%Ё̀Q=5Ѡ($$$$$%ѡQх̀Yѕ̤($$$$$$%ѡQх􁑅̀Yѕ($$$$$%($$$$%(($$$$% 1YѕÀ̠ 1YѕA̠($$$$$%̀􁝕 1YѕA̠($$$$%(($$$$%ѕЀ􁹕܁YѕQхM͡СQQхѡQхݕQхQх̰($$$$$$%ٽѕAYѕ̰ɕYѕAYѕIեɕѕ5ѡQх(($$$$%Ʌ1 յє􁹕܁Ʌ1($$$$%є܁ յQQх܁хYՕ%СQQх($$$$%є܁ յ5ѡQх܁хYՕ%СѡQх($$$$% Mѽɕ5ѡQх]ѡє($$$$$%є܁ յ5ѡQх]ѡѕAѠ܁хYՕ%Сѕ5ѡQх($$$$%($$$$%є܁ յ]Qх܁хYՕ%СݕQх($$$$%є܁ յQх܁хYՕ%СQх($$$$%є܁ յÄ܁хYՕ%С̤(($$$$%՜Mѥѽх̀ѕйѽMɥٽѕ%􈀬ٽѕ%ȀȀ($$$$$$$͕٥($$$$%Aɽ5ME0єեє($$$%􁕱͔($$$$%ѕЀ􁹕܁YѕQхM͡РٽѕAYѕ̰ɕYѕAYѕIեɕ($$$%($$%($$%ѕЀձ($$$%ѕЀ􁹕܁YѕQхM͡РٽѕAYѕ̰ɕYѕAYѕIեɕ($$%(($$%Yѕ1Mх́ٽѕMх̀Yѕ1Mх̹%55%Q($$%хAɽ ɽЀ􁍅YѕMх ɽЀɽ ɽ!($$$$%ɽ ɽȹ͕%ѕ݅ɑ=($$%MMɥɽ ɽQɝ̀ ѥ̹MР($$%хAɽ ɽФ($$$$ՕՕɽЁ݅́ͅɥͅݡѡȁ݅($$$$I䁽хɝ́ѡЁЁɕ٥ͱ䁅Ёٕ($$$%ɽ ɽQɝ̀ɽ ɽ!܁1!͡MɽQɝ̤($$$$$$ɽ ɽȹɕͽٕQɝ̡͔ձ($$$%MMɥɕQɝ̀􁹕܁1!͡Mɽ ɽQɝ̤($$$%ɕQɝ̹ɕٕɽ݅ɑMٕ̤($$$%ɽ݅ɑMٕ̹͕Aɽ ɽСɕQɝ̰եȰ͕٥ѥ($$$$$%ѕЀձѕйѽMɥ͔($$%(($$$($$$Mٽє̤Ѽ̤($$$($$% MYѕQMٕ̠($$$%ȀMɥ̀聝مMٕ̠(($$$$%ɍ 􁝕 ]U͕=($$$$$$$=MٕȀձMٕȹՅ%ɕ ͔̤(($$$$%ɍ ($$$$$%՜ɍٽєѼȁ͕ٕȀ̤($$$$%(($$$$%M=MٕYѕIѥ̤ѡɕեɕA=ɍ ($$$$$%ٽѕMх̀Yѕ1Mх̹  !($$$$$%ɽ݅ɑхAɽ ɽ($$$$$$$$ɽ݅ɑMٕ̹хɽ ɽQɝ̤($$$$$%Yѕ !ȠMٕYє̰($$$$$$$%܁= չYєٽѕ%Ȱե͕٥ѥɕYє($$$$$$$$$%ѕйѽMɥɽ݅ɑхAɽ ɽа($$$$$$$$$%ɽ ɽQɝ̰ɽ݅ɑMٕ̰͔($$$$$%՜ ٽєȀȀ͕٥Ȁ̤($$$$%􁕱͔($$$$$%ɽ!ɔ􀅉ɽ݅ɑMٕ̹х̡̤($$$$$%ɽ!ɔ Aɽ ɽ($$$$$$%MMɥхɝ̀хAɽ ɽЀɽ ɽQɝ($$$$$$$$$ɽ ɽȹɕͽٕQɝ̡=MٕȤ($$$$$$%ɽ!ɔɽ ɽȹ͡ձ ɽС̰хɝ̤($$$$$%(($$$$$%5ͅAɽ!ȹ͕5̰ͅȰ($$$$$$$%YѥA՝]ɔٽєȰե͕٥ѥՔɕYєѕйѽMɥ($$$$$$$$$%ٽѕ% չ5Qх̠ɽ!ɔİĤ($$$$%($$$%($$%􁕱͔($$$$M͕ٕȁ聽́Ѽȁ͕ٕ쁽ѡݥ͔ՕՔ̀($$$$ٽє($$$%=MٕȀձمMٕ̠х̡MٕȤ($$$$%Mɥ͕ٕȀMٕ(($$$$%ɽ!ɔ􀅉ɽ݅ɑMٕ̹х̡͕ٕȤ($$$$%ɽ!ɔ Aɽ ɽ($$$$$%MMɥхɝ̀хAɽ ɽЀɽ ɽQɝ($$$$$$$$ɽ ɽȹɕͽٕQɝ̡ՔMٕȤ($$$$$%ɽ!ɔɽ ɽȹ͡ձ ɽС͕ٕȰхɝ̤($$$$%(($$$$%5ͅAɽ!ȹ͕5͕ٕͅȰİ($$$$$$%YѥA՝]ɔٽѕ=Ȱե͕٥ѥՔɕYєѕйѽMɥ($$$$$$$$%ٽѕ% չ5Qх̠ɽ!ɔİĤ(($$$$%YѕMх ɽЀ Aɽ ɽ($$$$$$$хAɽ ɽФ($$$$$%MMɥхɝ̀ɽ ɽȹɕͽٕQɝ̡ՔMٕȤ(($$$$$%Ё($$$$$%ȀMɥхɝMٕȀхɝ̤($$$$$$$ٽՉɽЁѡ͕ٕͅȁѡЁɕ䁝Ёѡٽѕ=($$$$$$%хɝMٕȹՅ%ɕ ͔͕ٕȤ($$$$$$$%ѥՔ($$$$$$%($$$$$$% Mٕ̠х̡хɝMٕȤ($$$$$$$%ѥՔ($$$$$$%(($$$$$$%5ͅAɽ!ȹ͕5ͅхɝMٕȰ($$$$$$$$%YѥA՝]ɔٽѕ ɽСեȰ͕٥ѥ($$$$$$$$$$%ѕЀձѕйѽMɥՔ($$$$$$%䬬($$$$$%($$$$%(($$$$$ձѥɽ聕ٕ䁍ȁٽє($$$$% 5ձѥAɽMР 5ձѥAɽ=I݅ɐ($$$$$%ձѥAɽ!ȹ͕ YєեȤ($$$$%($$$%􁕱͔($$$$%ٽѕMх̀Yѕ1Mх̹  !($$$$%ɽ݅ɑхAɽ ɽ($$$$$$$ɽ݅ɑMٕ̹хɽ ɽQɝ̤($$$$%Yѕ !Ƞ=Yєե($$$$$$%܁= չYєٽѕ%Ȱե͕٥ѥɕYєѕйѽMɥ($$$$$$$$%ɽ݅ɑхAɽ ɽаɽ ɽQɝ̰($$$$$$$$%ɽ݅ɑMٕ̰͔($$$$%՜ ٽєȀȀ͕٥($$$%(($$$%Ё($$$%ȀMɥ̀聝مMٕ̠($$$$%5ͅAɽ!ȹ͕5̰ͅ䀬İYѥA՝]ɔٽѕUєե($$$$$$%ٽѕAYѕ̰ɕYѕAYѕIեɕ͕٥ѥѕйѽMɥ($$$$%䀬($$$%($$%(($$$Yє($$%ٽѕ15űQձ Yѕ1($$$%ٽѕ15űQYєٽѕ%ٽѕMх̰͕٥եȰѥ($$$$$%Yѕ !ȠAɽ Qхե($$%(($$$($$$5ձѥɽ䁙݅ɑ($$$($$% 5ձѥAɽMР AɥMٕȠ($$$% 5ձѥAɽ=I݅ɐ($$$$%՜Mɽٽєٕ($$$$%ձѥAɽ!ȹ͕5ձѥAɽٕYѥA՝]ɔٽєȰե͕٥ѥ͔($$$$$$%ɕYєѕЀձѕйѽMɥٽѕ%͔͔İĤ($$$%􁕱͔($$$$$=͕Ѽѡȁɽᥕ́ѡȁ%9=Pɕɕٔɕ݅ɐ($$$$$($$$$%͡ձMՔ($$$$%=MٕȀձ($$$$$% Mٕ̠х̡MٕȤ($$$$$$%͡ձM􁙅͔($$$$$%($$$$%(($$$$%͡ձM($$$$$%՜Mɽٽѕٕ($$$$$%ձѥAɽ!($$$$$$$$͕5ձѥAɽٕYѥA՝]ɔٽѕ=Ȱե͕٥ѥ͔($$$$$$$$$%ɕYєѕЀձѕйѽMɥٽѕ%͔͔İĤ($$$$%􁕱͔($$$$$%՜9Ё͕ɽ䁵ͅȁٽѕȁɕ䁝Ёɕ݅ɐ($$$$%($$$%($$%($$%ՕՕYєձ($$$%ՕՕYє͕Aɽ͕Ք($$$%Yѕ !ȠѕQYєՕՕYє($$$$%݅ɸUѼͥЁѕɽٕȁٽєՕՕYєYѕ%($$$$$$$쁅ѕѥɅɕمѕ䈤($$$%($$%($$%ɕɸEՕՕYѕIձйMU ML($%􁍅эፕѥ($$%ɥMхQɅ($$%ɕɸEՕՕYѕIձйIQId($%(%((%ɥمєхѥ́AAɕ͕!($%ɥمєUU%ɕՕ%($%ɥمєUU%Uե($%ɥمєMɥ9($%ɥمєMɥե($%ɥمєMɥ͕ٕ($%ɥمєUU%ѥ%($%ɥمєUU%%ɹѥ%($%ɥمєMхѕ($%ɥمєMՕ($%ɥمєɕѕ(($%ɥمєAAɕ͕!Mɥ9MɥեMɥ͕ٕȰUU%ѥ%($$$%UU%%ɹѥ%MхѕаMՕɕѕФ($$%ѡ̹Uե͕AUեե($$%ѡ̹99($$%ѡ̹եե($$%ѡ̹͕ٕȀ͕ٕ($$%ѡ̹ѥ%􁍽ѥ%($$%ѡ̹%ɹѥ%􁉅%ɹѥ%($$%ѡ̹MхѕЀ􁉅Mхѕ($$%ѡ̹MՕ􁍽MՕ($$%ѡ̹ɕѕЀɕѕ($%(($%ɥمєхѥUU%͕AUեMɥե($$%($$$%ɕɸUU%ɽMɥեɥ($$%􁍅эፕѥɕ($$$%ɕɸձ($$%($%(%((%ՉɅЁٽ݅ɸMɥͅ((%ՉɅЁMձᕍѽM٥MձȠ)( \ No newline at end of file +package com.bencodez.votingplugin.proxy; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; +import java.io.File; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.sql.SQLException; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.eclipse.paho.client.mqttv3.MqttException; + +import com.bencodez.advancedcore.api.time.TimeType; +import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalDataHandlerProxy; +import com.bencodez.advancedcore.bungeeapi.globaldata.GlobalMySQL; +import com.bencodez.advancedcore.bungeeapi.time.BungeeTimeChecker; +import com.bencodez.simpleapi.encryption.EncryptionHandler; +import com.bencodez.simpleapi.json.JsonParser; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageListener; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageProxyHandler; +import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; +import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; +import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; +import com.bencodez.simpleapi.servercomm.redis.RedisHandler; +import com.bencodez.simpleapi.servercomm.redis.RedisListener; +import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; +import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; +import com.bencodez.simpleapi.servercomm.sockets.SocketReceiver; +import com.bencodez.simpleapi.sql.Column; +import com.bencodez.simpleapi.sql.DataType; +import com.bencodez.simpleapi.sql.data.DataValue; +import com.bencodez.simpleapi.sql.data.DataValueBoolean; +import com.bencodez.simpleapi.sql.data.DataValueInt; +import com.bencodez.simpleapi.sql.data.DataValueString; +import com.bencodez.simpleapi.sql.mysql.config.MysqlConfig; +import com.bencodez.votingplugin.proxy.broadcast.ProxyBroadcastDecider; +import com.bencodez.votingplugin.proxy.cache.IVoteCache; +import com.bencodez.votingplugin.proxy.cache.VoteCacheHandler; +import com.bencodez.votingplugin.proxy.cache.nonvoted.INonVotedPlayersStorage; +import com.bencodez.votingplugin.proxy.cache.nonvoted.NonVotedPlayersCache; +import com.bencodez.votingplugin.proxy.control.ControlConnector; +import com.bencodez.votingplugin.proxy.control.HostedControlManager; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyHandler; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyMethod; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfiguration; +import com.bencodez.votingplugin.proxy.multiproxy.MultiProxyServerSocketConfigurationBungee; +import com.bencodez.votingplugin.proxy.presence.BackendPlayerPresenceTracker; +import com.bencodez.votingplugin.proxy.presence.PlayerPresence; +import com.bencodez.votingplugin.timequeue.VoteTimeQueue; +import com.bencodez.votingplugin.topvoter.TopVoter; +import com.bencodez.votingplugin.util.MinecraftUsernameValidator; +import com.bencodez.votingplugin.util.ServiceSiteValidator; +import com.bencodez.votingplugin.votelog.VoteLogMysqlTable; +import com.bencodez.votingplugin.votelog.VoteLogMysqlTable.VoteLogStatus; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; + +import redis.clients.jedis.DefaultJedisClientConfig; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; +import redis.clients.jedis.JedisPool; + +import lombok.Getter; +import lombok.Setter; + +public abstract class VotingPluginProxy { + private static final long PRESENCE_HANDOFF_TIMEOUT_MILLIS = TimeUnit.MINUTES.toMillis(2); + private static final long PRESENCE_STARTUP_RESYNC_DELAY_SECONDS = 5L; + private static final long PRESENCE_MAINTENANCE_INTERVAL_SECONDS = 30L; + private static final long PRESENCE_BACKEND_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(90); + + @Getter + @Setter + private int votePartyVotes = 0; + + @Getter + @Setter + private int currentVotePartyVotesRequired = 0; + + @Getter + @Setter + private ProxyMysqlUserTable proxyMySQL; + + private EncryptionHandler encryptionHandler; + + private HashMap clientHandles; + + private SocketHandler socketHandler; + + @Getter + @Setter + private boolean votifierEnabled = true; + + @Getter + private ConcurrentHashMap uuidPlayerNameCache = new ConcurrentHashMap<>(); + + @Getter + @Setter + private GlobalDataHandlerProxy globalDataHandler; + + @Getter + private RedisHandler redisHandler; + private JedisPool redisPublisherPool; + private volatile long redisPublisherRetryAfter; + private boolean timeVoteRetryScheduled; + private boolean timeVoteDeliveryRetryScheduled; + private boolean cachedVoteDeliveryRetryScheduled; + + private boolean enabled; + + @Getter + @Setter + private MultiProxyHandler multiProxyHandler; + + @Getter + private BungeeTimeChecker bungeeTimeChecker; + + @Getter + @Setter + private BungeeMethod method; + + @Getter + private MqttHandler mqttHandler; + + @Getter + private GlobalMessageProxyHandler globalMessageProxyHandler; + + @Getter + @Setter + private MySqlMessenger proxyMysqlMessenger; + + @Getter + private VoteCacheHandler voteCacheHandler; + + @Getter + private NonVotedPlayersCache nonVotedPlayersCache; + + @Getter + private final BackendPlayerPresenceTracker backendPlayerPresenceTracker = new BackendPlayerPresenceTracker(); + private final Map pendingPresenceHandoffs = new HashMap<>(); + private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); + private volatile ControlConnector controlConnector; + private volatile HostedControlManager hostedControlManager; + + public VotingPluginProxy() { + enabled = true; + + bungeeTimeChecker = new BungeeTimeChecker(getConfig().getTimeZone(), getConfig().getTimeHourOffSet(), + getConfig().getTimeWeekOffSet()) { + + @Override + public void debug(String text) { + debug2(text); + } + + @Override + public long getLastUpdated() { + return getVoteCacheLastUpdated(); + } + + @Override + public int getPrevDay() { + return getVoteCachePrevDay(); + } + + @Override + public String getPrevMonth() { + return getVoteCachePrevMonth(); + } + + @Override + public int getPrevWeek() { + return getVoteCachePrevWeek(); + } + + @Override + public void info(String text) { + log(text); + } + + @Override + public boolean isEnabled() { + return enabled; + } + + @Override + public boolean isIgnoreTime() { + return isVoteCacheIgnoreTime(); + } + + @Override + public void setIgnoreTime(boolean ignore) { + setVoteCacheVoteCacheIgnoreTime(ignore); + } + + @Override + public void setLastUpdated() { + setVoteCacheLastUpdated(); + } + + @Override + public void setPrevDay(int day) { + setVoteCachePrevDay(day); + } + + @Override + public void setPrevMonth(String text) { + setVoteCachePrevMonth(text); + } + + @Override + public void setPrevWeek(int week) { + setVoteCachePrevWeek(week); + } + + @Override + public void timeChanged(TimeType type, boolean fake, boolean pre, boolean post) { + if (getConfig().getVoteCacheTime() > 0) { + getVoteCacheHandler().checkVoteCacheTime(getConfig().getVoteCacheTime()); + } + if (!getConfig().getGlobalDataEnabled()) { + warn("Global data not enabled, ignoring time change event"); + return; + } + int delay = 1; + for (String s : getAllAvailableServers()) { + if (getGlobalDataHandler().getGlobalMysql().containsKey(s)) { + String lastOnlineStr = getGlobalDataHandler().getString(s, "LastOnline"); + long lastOnline = 0; + try { + lastOnline = Long.valueOf(lastOnlineStr); + } catch (NumberFormatException e) { + // ignore + } + + if (LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli() - lastOnline < 1000 + * 60 * 60 * 12) { + HashMap dataToSet = new HashMap<>(); + dataToSet.put("LastUpdated", new DataValueString( + "" + LocalDateTime.now().atZone(ZoneOffset.UTC).toInstant().toEpochMilli())); + dataToSet.put("FinishedProcessing", new DataValueBoolean(false)); + dataToSet.put(type.toString(), new DataValueBoolean(true)); + getGlobalDataHandler().setData(s, dataToSet); + + globalMessageProxyHandler.sendMessage(s, delay, VotingPluginWire.bungeeTimeChange()); + delay++; + } else { + warn("Server " + s + " hasn't been online recently"); + } + } else { + warn("Server " + s + " global data handler disabled?"); + } + } + globalDataHandler.onTimeChange(type); + } + + @Override + public void warning(String text) { + warn(text); + } + }; + } + + public void onTimeChangedFailed(String srv, TimeType type) { + getGlobalDataHandler().setBoolean(srv, type.toString(), false); + getGlobalDataHandler().setBoolean(srv, "FinishedProcessing", true); + getGlobalDataHandler().setBoolean(srv, "Processing", false); + } + + public void onTimeChangedFinished(TimeType type) { + if (type.equals(TimeType.MONTH)) { + getProxyMySQL().copyColumnData(TopVoter.Monthly.getColumnName(), "LastMonthTotal"); + } + getProxyMySQL().wipeColumnData(TopVoter.of(type).getColumnName(), DataType.INTEGER); + + if (!getConfig().getGlobalDataEnabled()) { + return; + } + for (String s : getAllAvailableServers()) { + getGlobalDataHandler().setBoolean(s, "ForceUpdate", true); + getGlobalMessageProxyHandler().sendMessage(s, 1, VotingPluginWire.bungeeTimeChange()); + } + processQueue(); + } + + /** + * Load MySQL + global data handler. + */ + public void loadMysql(MysqlConfig mysqlConfig, MysqlConfig globalDataMysqlConfig) { + if (mysqlConfig.getHostName().isEmpty() || mysqlConfig.getDatabase().isEmpty()) { + logSevere("MySQL is not configured correctly. " + "Missing host/database. host=" + mysqlConfig.getHostName() + + " db=" + mysqlConfig.getDatabase()); + setProxyMySQL(null); + return; + } + + setProxyMySQL(new ProxyMysqlUserTable("VotingPlugin_Users", mysqlConfig, getConfig().getDebug()) { + + @Override + public void debug(SQLException e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable t) { + if (getConfig().getDebug()) { + t.printStackTrace(); + } + } + + @Override + public void debug(String str) { + debug2(str); + } + }); + + ArrayList servers = new ArrayList(getAllAvailableServers()); + + if (getConfig().getGlobalDataEnabled()) { + if (getConfig().getGlobalDataUseMainMySQL()) { + setGlobalDataHandler(new GlobalDataHandlerProxy( + new GlobalMySQL("VotingPlugin_GlobalData", getProxyMySQL().getMysql()) { + + @Override + public void debugEx(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debugLog(String text) { + debug(text); + } + + @Override + public void info(String text) { + logInfo(text); + } + + @Override + public void logSevere(String text) { + VotingPluginProxy.this.logSevere(text); + } + + @Override + public void warning(String text) { + warn(text); + } + }, servers) { + + @Override + public void onTimeChangedFailed(String srv, TimeType type) { + VotingPluginProxy.this.onTimeChangedFailed(srv, type); + } + + @Override + public void onTimeChangedFinished(TimeType type) { + VotingPluginProxy.this.onTimeChangedFinished(type); + } + }); + } else { + setGlobalDataHandler( + new GlobalDataHandlerProxy(new GlobalMySQL("VotingPlugin_GlobalData", globalDataMysqlConfig) { + + @Override + public void debugEx(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debugLog(String text) { + debug(text); + } + + @Override + public void info(String text) { + logInfo(text); + } + + @Override + public void logSevere(String text) { + VotingPluginProxy.this.logSevere(text); + } + + @Override + public void warning(String text) { + warn(text); + } + }, servers) { + + @Override + public void onTimeChangedFailed(String srv, TimeType type) { + VotingPluginProxy.this.onTimeChangedFailed(srv, type); + } + + @Override + public void onTimeChangedFinished(TimeType type) { + VotingPluginProxy.this.onTimeChangedFinished(type); + } + }); + } + + // update global schema columns (unchanged from original) + getGlobalDataHandler().getGlobalMysql().alterColumnType("IgnoreTime", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("MONTH", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("WEEK", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("DAY", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("FinishedProcessing", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("Processing", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("ForceUpdate", "VARCHAR(5)"); + getGlobalDataHandler().getGlobalMysql().alterColumnType("LastUpdated", "MEDIUMTEXT"); + } + + // column types (unchanged from original) + getProxyMySQL().alterColumnType("TopVoterIgnore", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("CheckWorld", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("Reminded", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("DisableBroadcast", "VARCHAR(5)"); + getProxyMySQL().alterColumnType("LastOnline", "VARCHAR(20)"); + getProxyMySQL().alterColumnType("PlayerName", "VARCHAR(30)"); + getProxyMySQL().alterColumnType("DailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("WeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("DayVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("BestDayVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("WeekVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("BestWeekVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("VotePartyVotes", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("MonthVoteStreak", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("Points", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestDailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("AllTimeTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestMonthlyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("MonthTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("HighestWeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastMonthTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastWeeklyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("LastDailyTotal", "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType("OfflineRewards", "MEDIUMTEXT"); + getProxyMySQL().alterColumnType("DayVoteStreakLastUpdate", "MEDIUMTEXT"); + + if (getConfig().getStoreMonthTotalsWithDate()) { + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now()), "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(1)), + "INT DEFAULT '0'"); + getProxyMySQL().alterColumnType(getMonthTotalsWithDatePath(LocalDateTime.now().plusMonths(2)), + "INT DEFAULT '0'"); + } + } + + public void addCurrentVotePartyVotes(int amount) { + votePartyVotes += amount; + setVoteCacheVotePartyCurrentVotes(votePartyVotes); + debug("Current vote party total: " + votePartyVotes); + } + + public void addNonVotedPlayer(String uuid, String playerName) { + nonVotedPlayersCache.addPlayer(uuid, playerName); + } + + public void addVoteParty() { + if (getConfig().getVotePartyEnabled()) { + addCurrentVotePartyVotes(1); + checkVoteParty(); + } + } + + public abstract void broadcast(String message); + + private Set sendProxyBroadcast(Set targets, String uuid, String player, String service, long time, + String text, boolean wasOnline) { + Set forwarded = new LinkedHashSet<>(); + for (String targetServer : targets) { + JsonEnvelope envelope = VotingPluginWire.voteBroadcast(uuid, player, service, time, text, wasOnline); + if (sendProxyBroadcastEnvelopeNow(targetServer, envelope)) { + forwarded.add(targetServer); + } + } + return forwarded; + } + + /** + * Sends a standalone proxy broadcast through the selected transport and reports + * whether that transport accepted the message. + * + * @param server target backend server + * @param envelope standalone broadcast envelope + * @return true only when the transport accepted the message + */ + protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope envelope) { + switch (method) { + case MQTT: + return sendMqttEnvelopeServer(server, envelope); + case MYSQL: + if (proxyMysqlMessenger == null) { + return false; + } + try { + proxyMysqlMessenger.sendToBackend(server, envelope); + return true; + } catch (SQLException e) { + debug(e.getMessage()); + return false; + } + case PLUGINMESSAGING: + return sendPluginMessageServerNow(server, envelope); + case REDIS: + return sendRedisEnvelopeServer(server, envelope, true); + case SOCKETS: + // Standalone broadcasts use the same initialized client as normal + // envelopes. This preserves the socket connection and its delivery + // acknowledgement instead of creating a second short-lived socket. + ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); + if (socketClient == null) { + return false; + } + try { + socketClient.sendEnvelope(envelope); + return true; + } catch (RuntimeException e) { + debug(e.getMessage()); + return false; + } + default: + return false; + } + } + + public synchronized void checkCachedVotes(String server) { + int delay = 1; + if (isServerValid(server)) { + if (isSomeoneOnlineServerForVoteRouting(server)) { + if (getVoteCacheHandler().hasVotes(server) && !getConfig().getBlockedServers().contains(server)) { + ArrayList c = getVoteCacheHandler().getVotes(server); + ArrayList removed = new ArrayList<>(); + if (!c.isEmpty()) { + int num = 1; + int numberOfVotes = c.size(); + for (OfflineBungeeVote cache : c) { + if (cache.isDeliveryStateDirty() && !persistServerVoteDelivery(server, cache)) { + continue; + } + if (cache.isProxyBroadcastHandled() && cache.needsBroadcastOn(server)) { + Set forwarded = sendProxyBroadcast(Collections.singleton(server), + cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), + cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (!persistServerVoteDelivery(server, cache)) { + continue; + } + } + } + + boolean toSend = true; + if (getConfig().getWaitForUserOnline()) { + if (!isPlayerOnlineForVoteRouting(cache.getPlayerName())) { + toSend = false; + } else if (isPlayerOnlineForVoteRouting(cache.getPlayerName()) + && !getCurrentPlayerServerForVoteRouting(cache.getPlayerName()).equals(server)) { + toSend = false; + } + } + if (toSend) { + boolean broadcastHere = cache.needsBroadcastOn(server); + if (!cache.isProxyBroadcastHandled() && broadcastHere + && getConfig().getProxyBroadcastEnabled()) { + boolean playerOnline = isPlayerOnlineForVoteRouting(cache.getPlayerName()); + String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(cache.getPlayerName()) + : null; + + Set targets = proxyBroadcastDecider.resolveTargets(playerOnline, + playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + globalMessageProxyHandler.sendMessage(server, delay, + VotingPluginWire.vote(cache.getPlayerName(), cache.getUuid(), + cache.getService(), cache.getTime(), false, cache.isRealVote(), + cache.getText(), cache.getVoteId(), getConfig().getBungeeManageTotals(), + broadcastHere, num, numberOfVotes)); + delay++; + num++; + removed.add(cache); + } else { + debug("Not sending vote because user isn't on server " + server + ": " + + cache.toString()); + } + } + getVoteCacheHandler().removeServerVotes(server, removed); + } else { + debug("No cached votes for server: " + server); + } + } else { + debug("No cached votes for server: " + server); + } + } + } else { + debug("Server not valid: " + server); + } + } + + public synchronized void checkOnlineVotes(String player, String uuid, String server) { + int delay = 1; + if (isPlayerOnlineForVoteRouting(player) && getVoteCacheHandler().hasOnlineVotes(uuid)) { + ArrayList c = getVoteCacheHandler().getOnlineVotes(uuid); + if (!c.isEmpty()) { + if (server == null) { + server = getCurrentPlayerServerForVoteRouting(player); + } + if (!getConfig().getBlockedServers().contains(server)) { + int num = 1; + int numberOfVotes = (int) c.stream().filter(vote -> !vote.isRewardDelivered()).count(); + boolean deliveredReward = false; + ArrayList retained = new ArrayList<>(); + for (OfflineBungeeVote cache : c) { + if (cache.isProxyBroadcastHandled()) { + Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); + pendingTargets.removeAll(cache.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + cache.getBroadcastForwardedServers().addAll(sendProxyBroadcast(pendingTargets, + cache.getUuid(), cache.getPlayerName(), cache.getService(), cache.getTime(), + cache.getText(), false)); + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + } + boolean broadcastHere = cache.needsBroadcastOn(server); + if (!cache.isProxyBroadcastHandled() && broadcastHere + && getConfig().getProxyBroadcastEnabled()) { + String playerServer = (server != null) ? server : getCurrentPlayerServerForVoteRouting(player); + + Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + if (!cache.isRewardDelivered()) { + globalMessageProxyHandler.sendMessage(server, delay, + VotingPluginWire.voteOnline(cache.getPlayerName(), cache.getUuid(), cache.getService(), + cache.getTime(), false, cache.isRealVote(), cache.getText(), cache.getVoteId(), + getConfig().getBungeeManageTotals(), broadcastHere, num, numberOfVotes)); + // The normal envelope is also a valid broadcast delivery for the + // current target. Record it so a previously pending standalone + // retry cannot announce the same vote again later. + if (cache.isProxyBroadcastHandled() && broadcastHere) { + cache.getBroadcastForwardedServers().add(server); + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + } + cache.setRewardDelivered(true); + deliveredReward = true; + delay++; + num++; + } + + if (cache.isProxyBroadcastHandled() && !cache.isProxyBroadcastComplete()) { + retained.add(cache); + } + } + getVoteCacheHandler().removeOnlineVotes(uuid); + for (OfflineBungeeVote pending : retained) { + getVoteCacheHandler().addOnlineVote(uuid, pending); + } + + // multiproxy: envelope-only + if (deliveredReward && getConfig().getMultiProxySupport() + && getConfig().getMultiProxyOneGlobalReward()) { + multiProxyHandler.sendClearVote(uuid, player); + } + } + } + } + } + + /** + * Retries voter-keyed standalone broadcasts when any player makes a target + * backend available as a plugin-message carrier. + * + * @param server backend server that gained a carrier + */ + protected synchronized void retryPendingOnlineBroadcasts(String server) { + List blockedServers = getConfig().getBlockedServers(); + if (server == null || (blockedServers != null && blockedServers.contains(server))) { + return; + } + for (String cachedUuid : getVoteCacheHandler().getOnlineVoteUUIDs()) { + for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { + if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { + continue; + } + if (!cache.isProxyBroadcastHandled() || !cache.needsBroadcastOn(server)) { + continue; + } + Set forwarded = sendProxyBroadcast(Collections.singleton(server), cache.getUuid(), + cache.getPlayerName(), cache.getService(), cache.getTime(), cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); + } else { + persistOnlineVoteDelivery(cachedUuid, cache); + } + } + } + } + } + + protected synchronized void retryPendingTimeBroadcasts(String server) { + List blockedServers = getConfig().getBlockedServers(); + if (server == null || (blockedServers != null && blockedServers.contains(server))) { + return; + } + if (getVoteCacheHandler().getTimeChangeQueue() == null) { + return; + } + for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { + if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { + continue; + } + if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty() || !vote.getBroadcastTargets().contains(server) + || vote.getBroadcastForwardedServers().contains(server)) { + continue; + } + Set forwarded = sendProxyBroadcast(Collections.singleton(server), vote.getUuid(), vote.getName(), + vote.getService(), vote.getTime(), vote.getTotals(), false); + if (vote.getBroadcastForwardedServers().addAll(forwarded)) { + persistTimeVoteDelivery(vote); + } + } + } + + /** + * Periodically retries every pending voter-keyed standalone broadcast. This is + * required for broker transports whose recovery does not produce a player-login + * carrier event. + */ + public synchronized void retryPendingOnlineBroadcasts() { + for (String cachedUuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { + for (OfflineBungeeVote cache : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(cachedUuid))) { + if (cache.isDeliveryStateDirty() && !persistOnlineVoteDelivery(cachedUuid, cache)) { + continue; + } + if (!cache.isProxyBroadcastHandled() || cache.isProxyBroadcastComplete()) { + continue; + } + Set pendingTargets = new LinkedHashSet<>(cache.getBroadcastTargets()); + pendingTargets.removeAll(cache.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + Set forwarded = sendProxyBroadcast(pendingTargets, cache.getUuid(), cache.getPlayerName(), + cache.getService(), cache.getTime(), cache.getText(), false); + if (cache.getBroadcastForwardedServers().addAll(forwarded)) { + cache.setBroadcastForwarded(cache.isProxyBroadcastComplete()); + if (cache.isRewardDelivered() && cache.isProxyBroadcastComplete()) { + getVoteCacheHandler().removeOnlineVote(cachedUuid, cache); + } else { + persistOnlineVoteDelivery(cachedUuid, cache); + } + } + } + } + retryPendingTimeBroadcasts(); + } + + public synchronized void retryPendingTimeBroadcasts() { + if (getVoteCacheHandler().getTimeChangeQueue() == null) { + return; + } + for (VoteTimeQueue vote : new ArrayList<>(getVoteCacheHandler().getTimeChangeQueue())) { + if (vote.isDeliveryStateDirty() && !persistTimeVoteDelivery(vote)) { + continue; + } + if (!vote.isProxyBroadcastHandled() || vote.getUuid().isEmpty()) { + continue; + } + Set pendingTargets = new LinkedHashSet<>(vote.getBroadcastTargets()); + pendingTargets.removeAll(vote.getBroadcastForwardedServers()); + List blockedServers = getConfig().getBlockedServers(); + if (blockedServers != null) { + pendingTargets.removeAll(blockedServers); + } + Set forwarded = sendProxyBroadcast(pendingTargets, vote.getUuid(), vote.getName(), vote.getService(), + vote.getTime(), vote.getTotals(), false); + if (vote.getBroadcastForwardedServers().addAll(forwarded)) { + persistTimeVoteDelivery(vote); + } + } + } + + protected synchronized boolean persistTimeVoteDelivery(VoteTimeQueue vote) { + if (getVoteCacheHandler().updateTimeVote(vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleTimeVoteDeliveryRetry(); + return false; + } + + private void scheduleTimeVoteDeliveryRetry() { + if (timeVoteDeliveryRetryScheduled || getScheduler() == null) { + return; + } + timeVoteDeliveryRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + timeVoteDeliveryRetryScheduled = false; + } + retryPendingTimeBroadcasts(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + timeVoteDeliveryRetryScheduled = false; + debug("Unable to schedule timed broadcast state retry: " + e.getMessage()); + } + } + + protected synchronized boolean persistServerVoteDelivery(String server, OfflineBungeeVote vote) { + if (getVoteCacheHandler().updateServerVote(server, vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleCachedVoteDeliveryRetry(); + return false; + } + + protected synchronized boolean persistOnlineVoteDelivery(String uuid, OfflineBungeeVote vote) { + if (getVoteCacheHandler().updateOnlineVote(uuid, vote)) { + vote.setDeliveryStateDirty(false); + return true; + } + vote.setDeliveryStateDirty(true); + scheduleCachedVoteDeliveryRetry(); + return false; + } + + private void scheduleCachedVoteDeliveryRetry() { + if (cachedVoteDeliveryRetryScheduled || getScheduler() == null) { + return; + } + cachedVoteDeliveryRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + cachedVoteDeliveryRetryScheduled = false; + } + retryCachedVoteDeliveryPersistence(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + cachedVoteDeliveryRetryScheduled = false; + debug("Unable to schedule cached broadcast state retry: " + e.getMessage()); + } + } + + private synchronized void retryCachedVoteDeliveryPersistence() { + for (String server : getVoteCacheHandler().getCachedVotesServers()) { + for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getVotes(server))) { + if (vote.isDeliveryStateDirty()) { + persistServerVoteDelivery(server, vote); + } + } + } + for (String uuid : new LinkedHashSet<>(getVoteCacheHandler().getOnlineVoteUUIDs())) { + for (OfflineBungeeVote vote : new ArrayList<>(getVoteCacheHandler().getOnlineVotes(uuid))) { + if (vote.isDeliveryStateDirty()) { + persistOnlineVoteDelivery(uuid, vote); + } + } + } + } + + public void checkVoteParty() { + if (getConfig().getVotePartyEnabled()) { + if (votePartyVotes >= currentVotePartyVotesRequired) { + debug("Vote party reached"); + addCurrentVotePartyVotes(-currentVotePartyVotesRequired); + + currentVotePartyVotesRequired += getConfig().getVotePartyIncreaseVotesRequired(); + setVoteCacheVotePartyIncreaseVotesRequired( + getVoteCacheVotePartyIncreaseVotesRequired() + getConfig().getVotePartyIncreaseVotesRequired()); + + if (!getConfig().getVotePartyBroadcast().isEmpty()) { + broadcast(getConfig().getVotePartyBroadcast()); + } + + for (String command : getConfig().getVotePartyBungeeCommands()) { + runConsoleCommand(command); + } + + if (getConfig().getVotePartySendToAllServers()) { + for (String server : getAllAvailableServers()) { + sendVoteParty(server); + } + } else { + for (String server : getConfig().getVotePartyServersToSend()) { + sendVoteParty(server); + } + } + } + saveVoteCacheFile(); + } + } + + public abstract void debug(String str); + + private void debug2(String message) { + debug(message); + } + + /** + * HTTP client used for Mojang API requests. + */ + private final HttpClient httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + + /** + * Fetches a player's UUID from the Mojang API. + * + * @param playerName player name + * @return player UUID, or {@code null} if not found + * @throws IOException if the request fails + * @throws InterruptedException if interrupted while waiting for the response + */ + public UUID fetchUUID(String playerName) throws IOException, InterruptedException { + if (playerName == null || playerName.equalsIgnoreCase("null")) { + return null; + } + + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create("https://api.mojang.com/users/profiles/minecraft/" + playerName)).GET() + .timeout(Duration.ofSeconds(5)).build(); + + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + + if (response.statusCode() == 400 || response.statusCode() == 404) { + log("There is no player with the name \"" + playerName + "\"!"); + return null; + } + + if (response.statusCode() < 200 || response.statusCode() >= 300) { + throw new IOException("Failed to fetch UUID for " + playerName + ", HTTP " + response.statusCode()); + } + + JsonElement element = JsonParser.parseString(response.body()); + if (element == null || !element.isJsonObject()) { + return null; + } + + JsonObject object = element.getAsJsonObject(); + if (!object.has("id") || object.get("id").isJsonNull()) { + return null; + } + + String uuidAsString = object.get("id").getAsString(); + return parseUUIDFromString(uuidAsString); + } + + public abstract Set getAllAvailableServers(); + + /** Complete platform server set before whitelist/blocked routing filters. */ + public abstract Set getAllConfiguredServers(); + + public abstract VotingPluginProxyConfig getConfig(); + + public abstract String getCurrentPlayerServer(String player); + + /** + * Resolves a player's server for vote routing. A dedicated voting proxy has no + * local players, so it uses the backend presence tracker instead. + */ + protected String getCurrentPlayerServerForVoteRouting(String player) { + if (isDedicatedVotingProxyEnabled()) { + return backendPlayerPresenceTracker.getPlayer(player).map(presence -> presence.getServer()).orElse(null); + } + return getCurrentPlayerServer(player); + } + + /** + * Dedicated routing is intentionally unavailable on plugin messaging: that + * transport is attached to a player-facing proxy and does not carry backend + * presence snapshots. + */ + protected boolean isDedicatedVotingProxyEnabled() { + return getConfig().getDedicatedVotingProxy() && method != null && method.supportsBackendPresence(); + } + + public abstract File getDataFolderPlugin(); + + public String getMonthTotalsWithDatePath() { + LocalDateTime cTime = getBungeeTimeChecker().getTime(); + return getMonthTotalsWithDatePath(cTime); + } + + public String getMonthTotalsWithDatePath(LocalDateTime cTime) { + return "MonthTotal-" + cTime.getMonth().toString() + "-" + cTime.getYear(); + } + + public abstract String getProperName(String uuid, String playerName); + + public abstract String getUUID(String playerName); + + private int getValue(ArrayList cols, String column, int toAdd) { + for (Column d : cols) { + if (d.getName().equalsIgnoreCase(column)) { + DataValue value = d.getValue(); + int num = 0; + if (value == null) { + return toAdd; + } + if (value.isInt()) { + num = value.getInt(); + } else if (value.isString()) { + try { + num = Integer.parseInt(value.getString()); + } catch (Exception e) { + // ignore + } + } + return num + toAdd; + } + } + return toAdd; + } + + private VoteTotalsSnapshot getProjectedRolloverTotals(ArrayList data, String player) { + List timeChanges = getGlobalDataHandler().getTimeChanges(); + boolean resetMonth = timeChanges.contains(TimeType.MONTH); + boolean resetWeek = timeChanges.contains(TimeType.WEEK); + boolean resetDay = timeChanges.contains(TimeType.DAY); + int acceptedQueuedVotes = 0; + int acceptedGlobalQueuedVotes = 0; + for (VoteTimeQueue queued : getVoteCacheHandler().getTimeChangeQueue()) { + if (!queued.isProcessed()) { + acceptedGlobalQueuedVotes++; + } + if (!queued.isProcessed() && queued.getName() != null && queued.getName().equalsIgnoreCase(player)) { + acceptedQueuedVotes++; + } + } + int voteIncrement = acceptedQueuedVotes + 1; + + int allTimeTotal = getValue(data, "AllTimeTotal", voteIncrement); + int monthTotal = resetMonth ? voteIncrement : getValue(data, "MonthTotal", voteIncrement); + int weeklyTotal = resetWeek ? voteIncrement : getValue(data, "WeeklyTotal", voteIncrement); + int dailyTotal = resetDay ? voteIncrement : getValue(data, "DailyTotal", voteIncrement); + int points = getValue(data, "Points", voteIncrement * getConfig().getPointsOnVote()); + + int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); + if (maxVotes > 0) { + int days = getBungeeTimeChecker().getTime().getDayOfMonth(); + if (monthTotal > days * maxVotes) { + monthTotal = days * maxVotes; + } + } + if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { + points = getConfig().getLimitVotePoints(); + } + + int dateMonthTotal = -1; + if (getConfig().getStoreMonthTotalsWithDate()) { + if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { + dateMonthTotal = resetMonth ? voteIncrement + : getValue(data, getMonthTotalsWithDatePath(), voteIncrement); + } else { + dateMonthTotal = monthTotal; + } + } + + int[] projectedVoteParty = getProjectedVotePartyState(acceptedGlobalQueuedVotes + 1); + return new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, + projectedVoteParty[0], projectedVoteParty[1], dateMonthTotal); + } + + protected boolean canForwardStandaloneBroadcast(boolean managesTotals) { + return managesTotals; + } + + protected int[] getProjectedVotePartyState(int acceptedVotes) { + int current = votePartyVotes; + int required = currentVotePartyVotesRequired; + if (!getConfig().getVotePartyEnabled()) { + return new int[] { current, required }; + } + + int increase = getConfig().getVotePartyIncreaseVotesRequired(); + for (int i = 0; i < acceptedVotes; i++) { + current++; + if (current >= required) { + current -= required; + required += increase; + } + } + return new int[] { current, required }; + } + + public abstract String getPluginVersion(); + + public abstract int getVoteCacheCurrentVotePartyVotes(); + + public abstract long getVoteCacheLastUpdated(); + + public abstract int getVoteCachePrevDay(); + + public abstract String getVoteCachePrevMonth(); + + public abstract int getVoteCachePrevWeek(); + + public abstract int getVoteCacheVotePartyIncreaseVotesRequired(); + + public abstract boolean isPlayerOnline(String playerName); + + /** + * Checks online state for vote routing, using backend presence only when this + * proxy is explicitly configured as the dedicated voting proxy. + */ + protected boolean isPlayerOnlineForVoteRouting(String playerName) { + return isDedicatedVotingProxyEnabled() ? backendPlayerPresenceTracker.getPlayer(playerName).isPresent() + : isPlayerOnline(playerName); + } + + public abstract boolean isServerValid(String server); + + public abstract boolean isSomeoneOnlineServer(String server); + + protected boolean isSomeoneOnlineServerForVoteRouting(String server) { + if (!isDedicatedVotingProxyEnabled()) { + return isSomeoneOnlineServer(server); + } + com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus status = backendPlayerPresenceTracker + .getBackendStatus(server); + return status != null && status.isAvailable() && status.getPlayerCount() > 0; + } + + public abstract boolean isVoteCacheIgnoreTime(); + + public abstract MysqlConfig getVoteCacheMySQLConfig(); + + public abstract MysqlConfig getNonVotedCacheMySQLConfig(); + + public abstract MysqlConfig getVoteLoggingMySQLConfig(); + + /** + * Shutdown MySQL-related resources safely. + */ + public void shutdownMySql() { + if (getProxyMysqlMessenger() != null) { + getProxyMysqlMessenger().shutdown(); + setProxyMysqlMessenger(null); + } + + if (getProxyMySQL() != null) { + getProxyMySQL().shutdown(); + setProxyMySQL(null); + } + } + + public void load(IVoteCache jsonStorage, INonVotedPlayersStorage nonVotedCacheJson) { + method = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (getMethod() == null) { + method = BungeeMethod.PLUGINMESSAGING; + } + warnUnsupportedDedicatedVotingProxyMode(); + uuidPlayerNameCache = getProxyMySQL().getRowsUUIDNameQuery(); + + bungeeTimeChecker.setTimeChangeFailSafeBypass(getConfig().getTimeChangeFailSafeBypass()); + bungeeTimeChecker.loadTimer(); + + voteCacheHandler = new VoteCacheHandler(getVoteCacheMySQLConfig(), getConfig().getVoteCacheUseMySQL(), + getConfig().getVoteCacheUseMainMySQL(), getProxyMySQL().getMysql(), getConfig().getDebug(), + jsonStorage) { + + @Override + public void logInfo1(String msg) { + logInfo(msg); + } + + @Override + public void logSevere1(String msg) { + logSevere(msg); + } + + @Override + public void debug1(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debug1(String msg) { + if (getConfig().getDebug()) { + debug(msg); + } + } + + @Override + public void debug1(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + }; + voteCacheHandler.load(); + + nonVotedPlayersCache = new NonVotedPlayersCache(getNonVotedCacheMySQLConfig(), + getConfig().getNonVotedCacheUseMySQL(), getConfig().getNonVotedCacheUseMainMySQL(), + getProxyMySQL().getMysql(), nonVotedCacheJson, getConfig().getDebug()) { + + @Override + public boolean userExists(String uuid) { + return getProxyMySQL().containsKeyQuery(uuid); + } + + @Override + public void logInfo1(String msg) { + logInfo(msg); + } + + @Override + public void logSevere1(String msg) { + logSevere(msg); + } + + @Override + public void debug1(Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public void debug1(String msg) { + if (getConfig().getDebug()) { + debug(msg); + } + } + + @Override + public Set getAllUUIDs() { + return getProxyMySQL().getUuids(); + } + }; + + if (method.equals(BungeeMethod.MYSQL)) { + try { + proxyMysqlMessenger = new MySqlMessenger("VotingPlugin", + getProxyMySQL().getMysql().getConnectionManager().getDataSource(), MySqlMessenger.Mode.PROXY, + null, // no serverId in PROXY mode + msg -> { + if (getConfig().getDebug()) { + debug("Got from " + msg.source + ": " + msg.envelope.getSubChannel() + " " + + msg.envelope.getFields()); + } + globalMessageProxyHandler.onMessage(msg.envelope); + }); + } catch (SQLException e) { + e.printStackTrace(); + } + } else if (method.equals(BungeeMethod.PLUGINMESSAGING)) { + if (getConfig().getPluginMessageEncryption()) { + encryptionHandler = new EncryptionHandler("VotingPlugin", + new File(getDataFolderPlugin(), "secretkey.key")); + } + } else if (method.equals(BungeeMethod.SOCKETS)) { + encryptionHandler = new EncryptionHandler("VotingPlugin", new File(getDataFolderPlugin(), "secretkey.key")); + + socketHandler = new SocketHandler(getPluginVersion(), getConfig().getBungeeHost(), + getConfig().getBungeePort(), encryptionHandler, getConfig().getDebug()) { + + @Override + public void log(String str) { + logInfo(str); + } + }; + + socketHandler.add(new SocketReceiver() { + @Override + public void onReceiveEnvelope(JsonEnvelope envelope) { + globalMessageProxyHandler.onMessage(envelope); + } + }); + + clientHandles = new HashMap<>(); + List l = getConfig().getBlockedServers(); + for (String s : getConfig().getSpigotServers()) { + if (!l.contains(s)) { + Map d = getConfig().getSpigotServerConfiguration(s); + String host = ""; + if (d.containsKey("Host")) { + host = (String) d.get("Host"); + } + int port = 1298; + if (d.containsKey("Port")) { + port = (int) d.get("Port"); + } + clientHandles.put(s, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); + } + } + } else if (method.equals(BungeeMethod.REDIS)) { + redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), + getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex()) { + + @Override + public void debug(String message) { + debug2(message); + } + }; + redisPublisherPool = new JedisPool(new HostAndPort(getConfig().getRedisHost(), getConfig().getRedisPort()), + buildRedisClientConfig()); + + runAsync(() -> { + RedisListener listener = redisHandler.createEnvelopeListener( + getConfig().getRedisPrefix() + "VotingPlugin", + (ch, env) -> globalMessageProxyHandler.onMessage(env)); + redisHandler.loadListener(listener); + }); + + } else if (method.equals(BungeeMethod.MQTT)) { + try { + mqttHandler = new MqttHandler(new MqttServerComm(getConfig().getMqttClientID(), + getConfig().getMqttBrokerURL(), getConfig().getMqttUsername(), getConfig().getMqttPassword()), + 2); + + mqttHandler.subscribeEnvelopes(getConfig().getMqttPrefix() + "votingplugin/servers/proxy", + (topic, env) -> globalMessageProxyHandler.onMessage(env)); + + } catch (MqttException e) { + e.printStackTrace(); + } catch (Exception e) { + e.printStackTrace(); + } + } + + currentVotePartyVotesRequired = getConfig().getVotePartyVotesRequired() + + getVoteCacheVotePartyIncreaseVotesRequired(); + votePartyVotes = getVoteCacheCurrentVotePartyVotes(); + + globalMessageProxyHandler = new GlobalMessageProxyHandler() { + @Override + public void sendMessage(String server, int delay, JsonEnvelope envelope) { + switch (method) { + case MQTT: + sendMqttEnvelopeServer(server, envelope); + break; + case MYSQL: + try { + proxyMysqlMessenger.sendToBackend(server, envelope); + } catch (SQLException e) { + e.printStackTrace(); + } + break; + case PLUGINMESSAGING: + sendPluginMessageServer(server, delay, envelope); + break; + case REDIS: + sendRedisEnvelopeServer(server, envelope); + break; + case SOCKETS: + ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); + if (socketClient != null) { + socketClient.sendEnvelope(envelope); + } + break; + default: + break; + } + } + }; + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGIN) { + @Override + public void onReceive(JsonEnvelope message) { + handleLoginMessage(message); + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_LOGOUT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); + if (!isPresenceServerValid(event.server, VotingPluginWire.SUB_LOGOUT) + || !isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, + event.presenceTimestamp, + VotingPluginWire.SUB_LOGOUT)) { + return; + } + if (!backendPlayerPresenceTracker.playerOffline(event.uuid, event.server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, + System.currentTimeMillis())) { + debug("Ignored invalid or stale logout envelope: " + message.getFields()); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STARTED) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STARTED) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_STARTED)) { + if (backendPlayerPresenceTracker.backendStarted(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis())) { + discardPendingPresenceHandoffs(server); + pendingBackendRecoverySnapshots.add(presenceServerKey(server)); + requestBackendPresenceSnapshot(server); + } + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_STOPPED) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_STOPPED) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_STOPPED)) { + if (backendPlayerPresenceTracker.backendStopped(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis())) { + discardPendingPresenceHandoffs(server); + pendingBackendRecoverySnapshots.remove(presenceServerKey(server)); + } + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_BACKEND_HEARTBEAT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + UUID backendIncarnationId = VotingPluginWire.readBackendIncarnationId(message); + long backendStartedAt = VotingPluginWire.readBackendStartedAt(message); + long presenceTimestamp = VotingPluginWire.readPresenceTimestamp(message); + if (isPresenceServerValid(server, VotingPluginWire.SUB_BACKEND_HEARTBEAT) + && isPresenceGenerationValid(backendIncarnationId, backendStartedAt, presenceTimestamp, + VotingPluginWire.SUB_BACKEND_HEARTBEAT)) { + backendPlayerPresenceTracker.heartbeat(server, backendIncarnationId, backendStartedAt, + presenceTimestamp, System.currentTimeMillis()); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener(VotingPluginWire.SUB_PRESENCE_SNAPSHOT) { + @Override + public void onReceive(JsonEnvelope message) { + if (!method.supportsBackendPresence()) { + return; + } + String server = message.getFields().getOrDefault(VotingPluginWire.K_SERVER, ""); + if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT)) { + return; + } + VotingPluginWire.PresenceSnapshot snapshot = VotingPluginWire.readPresenceSnapshot(message); + long now = System.currentTimeMillis(); + boolean accepted = snapshot.valid + && isPresenceGenerationValid(snapshot.backendIncarnationId, snapshot.backendStartedAt, + snapshot.presenceTimestamp, + VotingPluginWire.SUB_PRESENCE_SNAPSHOT) + && backendPlayerPresenceTracker.applySnapshotChunk(snapshot.server, + snapshot.requestId, snapshot.chunkIndex, snapshot.chunkCount, snapshot.players, + snapshot.backendIncarnationId, snapshot.backendStartedAt, + snapshot.presenceTimestamp, now); + if (!accepted) { + debug("Ignored invalid or unexpected presence snapshot from " + snapshot.server); + if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { + discardPendingPresenceHandoffs(snapshot.requestId); + } + } else if (backendPlayerPresenceTracker.getPendingSnapshotRequestId(snapshot.server, now) == null) { + pendingBackendRecoverySnapshots.remove(presenceServerKey(snapshot.server)); + Set handoffPlayers = completePendingPresenceHandoffs(snapshot.requestId, snapshot.server, + snapshot.backendIncarnationId, snapshot.backendStartedAt, now); + processDedicatedSnapshotLogins(snapshot.server, handoffPlayers); + } + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener("statusokay") { + @Override + public void onReceive(JsonEnvelope message) { + String server = message.getFields().getOrDefault("server", ""); + log("Status okay for " + server); + } + }); + + globalMessageProxyHandler.addListener(new GlobalMessageListener("voteupdate") { + @Override + public void onReceive(JsonEnvelope message) { + int delay = 1; + for (String send : getAllAvailableServers()) { + globalMessageProxyHandler.sendMessage(send, delay, message); + delay++; + } + } + }); + + proxyBroadcastDecider = new ProxyBroadcastDecider(() -> getConfig(), () -> getAllAvailableServers(), + s -> isServerValid(s), + s -> getConfig().getBlockedServers() != null && getConfig().getBlockedServers().contains(s)); + + loadMultiProxySupport(); + loadVoteLoggingMySQL(); + if (method.supportsBackendPresence()) { + scheduleBackendPresenceStartupResync(); + loadTaskTimer(this::maintainBackendPresence, PRESENCE_MAINTENANCE_INTERVAL_SECONDS, + PRESENCE_MAINTENANCE_INTERVAL_SECONDS); + } + startControlServices(); + + debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); + } + + private synchronized void startControlServices() { + stopControlServices(true); + if (getConfig().getControlHostedEnabled()) { + try { + hostedControlManager = HostedControlManager.create(this); + if (hostedControlManager != null) { + hostedControlManager.start(); + } + } catch (IllegalArgumentException e) { + hostedControlManager = null; + logSevere("[Control Host] configuration is invalid; VotingPlugin remains unaffected"); + } + } + if (!getConfig().getControlEnabled()) { + return; + } + try { + controlConnector = ControlConnector.create(this); + if (controlConnector != null) { + controlConnector.start(); + } + } catch (IOException | IllegalArgumentException e) { + controlConnector = null; + logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); + } + } + + private synchronized void stopControlServices(boolean waitForHosted) { + ControlConnector connector = controlConnector; + controlConnector = null; + if (connector != null) { + connector.close(); + } + HostedControlManager manager = hostedControlManager; + hostedControlManager = null; + if (manager != null) { + if (waitForHosted) { + manager.closeAndWait(); + } else { + manager.close(); + } + } + } + + public String getControlConnectorStatus() { + ControlConnector connector = controlConnector; + return connector == null ? "DISABLED" : connector.status().name(); + } + + public String getHostedControlStatus() { + HostedControlManager manager = hostedControlManager; + return manager == null ? "DISABLED" : manager.status().name(); + } + + /** + * Handles both the original login notification and extended presence logins. + * Kept protected so transport-policy behavior can be regression tested without + * initializing a live proxy transport. + * + * @param message login envelope + */ + protected void handleLoginMessage(JsonEnvelope message) { + VotingPluginWire.PlayerPresenceEvent event = VotingPluginWire.readPlayerPresenceEvent(message); + String player = event.player; + String uuid = event.uuid; + String server = event.server; + + if (player.isEmpty() || uuid.isEmpty()) { + logSevere("Invalid login envelope received: " + message.getFields()); + return; + } + boolean legacy = event.connectionId == null && event.backendIncarnationId == null + && event.backendStartedAt == 0L && event.presenceTimestamp == 0L; + boolean accepted = false; + String deliveryServer = server; + if (legacy) { + if (method == BungeeMethod.PLUGINMESSAGING) { + String proxyServer = getCurrentPlayerServer(player); + accepted = isLegacyLoginDestinationAuthoritative(player, uuid, proxyServer); + if (accepted) { + deliveryServer = proxyServer; + } + } else if (method != null && method.supportsBackendPresence() + && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN)) { + accepted = isLegacyLoginDestinationAuthoritative(player, uuid, server); + } + } else if (method != null && method.supportsBackendPresence() && event.connectionId != null + && isPresenceServerValid(server, VotingPluginWire.SUB_LOGIN) + && isPresenceGenerationValid(event.backendIncarnationId, event.backendStartedAt, + event.presenceTimestamp, VotingPluginWire.SUB_LOGIN)) { + BackendPlayerPresenceTracker.PlayerOnlineResult result = backendPlayerPresenceTracker.playerOnlineResult( + player, uuid, server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, event.presenceTimestamp, + System.currentTimeMillis()); + accepted = result.isAccepted(); + if (result.isConflictingPresence()) { + requestBackendPresenceSnapshot(server, + new PendingPresenceHandoff(player, uuid, server, event.connectionId, + event.backendIncarnationId, event.backendStartedAt, + result.getConflictSequence(), System.currentTimeMillis())); + } + } + + debug("Login: " + player + "/" + uuid + " " + server); + if (accepted) { + discardPendingPresenceHandoff(uuid); + login(player, uuid, deliveryServer); + } else { + debug("Ignored invalid or stale login envelope: " + message.getFields()); + } + } + + /** + * Validates a legacy login against an authority independent of the envelope. + * Player-facing proxies use their native live route and UUID. A dedicated + * voting proxy has no native player session, so it requires an exact modern + * presence match for the claimed destination. + */ + private boolean isLegacyLoginDestinationAuthoritative(String player, String uuid, String server) { + if (server == null || server.isBlank()) { + return false; + } + + UUID claimedUuid; + try { + claimedUuid = UUID.fromString(uuid.trim()); + } catch (RuntimeException e) { + return false; + } + + if (isDedicatedVotingProxyEnabled()) { + PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(player).orElse(null); + return presence != null && presence.getServer().equalsIgnoreCase(server) + && (!getConfig().getOnlineMode() || presence.getUuid().equals(claimedUuid)); + } + + if (!isPlayerOnline(player)) { + return false; + } + String proxyServer = getCurrentPlayerServer(player); + if (proxyServer == null || !proxyServer.equalsIgnoreCase(server)) { + return false; + } + if (!getConfig().getOnlineMode()) { + return true; + } + + String authoritativeUuid = getUUID(player); + if (authoritativeUuid == null || authoritativeUuid.isBlank()) { + return false; + } + try { + return claimedUuid.equals(UUID.fromString(authoritativeUuid.trim())); + } catch (IllegalArgumentException e) { + return false; + } + } + + private VoteLogMysqlTable voteLogMysqlTable; + + @Getter + private ProxyBroadcastDecider proxyBroadcastDecider; + + public void loadVoteLoggingMySQL() { + if (getConfig().getVoteLoggingEnabled()) { + if (getConfig().getVoteLoggingUseMainMySQL()) { + voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getProxyMySQL().getMysql(), + getVoteLoggingMySQLConfig(), getConfig().getDebug()) { + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public String getServerName() { + return "Proxy"; + } + }; + } else { + voteLogMysqlTable = new VoteLogMysqlTable("votingplugin_votelog", getVoteLoggingMySQLConfig(), + getConfig().getDebug()) { + + @Override + public void logSevere(String string) { + VotingPluginProxy.this.logSevere(string); + } + + @Override + public void logInfo(String string) { + VotingPluginProxy.this.logInfo(string); + } + + @Override + public void debug(Throwable e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + + @Override + public String getServerName() { + return "Proxy"; + } + }; + } + + if (getConfig().getVoteLoggingPurgeDays() > 0) { + loadTaskTimer(() -> voteLogMysqlTable.purgeOlderThanDays(getConfig().getVoteLoggingPurgeDays(), 100), + 60, 60 * 60); + } + + debug("Vote logging MySQL enabled"); + } else { + debug("Vote logging MySQL disabled"); + } + } + + public abstract void loadTaskTimer(Runnable runnable, long delaySeconds, long repeatSeconds); + + public void loadMultiProxySupport() { + if (multiProxyHandler != null) { + multiProxyHandler.close(); + } + multiProxyHandler = new MultiProxyHandler() { + + @Override + public void addNonVotedPlayerCache(String uuid, String player) { + addNonVotedPlayer(uuid, player); + } + + @Override + public void clearVote(String uuid) { + getVoteCacheHandler().clearOnlineVoteRewards(uuid); + } + + @Override + public boolean getDebug() { + return getConfig().getDebug(); + } + + @Override + public EncryptionHandler getEncryptionHandler() { + return encryptionHandler; + } + + @Override + public MultiProxyMethod getMultiProxyMethod() { + return MultiProxyMethod.getByName(getConfig().getMultiProxyMethod()); + } + + @Override + public String getMultiProxyPassword() { + return getConfig().getMultiProxyRedisPassword(); + } + + @Override + public String getMultiProxyRedisHost() { + return getConfig().getMultiProxyRedisHost(); + } + + @Override + public int getMultiProxyRedisPort() { + return getConfig().getMultiProxyRedisPort(); + } + + @Override + public int getMultiProxyRedisDbIndex() { + return getConfig().getMultiProxyRedisDbIndex(); + } + + @Override + public boolean getMultiProxyRedisUseExistingConnection() { + return getConfig().getMultiProxyRedisUseExistingConnection(); + } + + @Override + public String getMultiProxyServerName() { + return getConfig().getProxyServerName(); + } + + @Override + public Collection getMultiProxyServers() { + return getConfig().getMultiProxyServers(); + } + + @Override + public MultiProxyServerSocketConfiguration getMultiProxyServersConfiguration(String s) { + return new MultiProxyServerSocketConfigurationBungee(s, + getConfig().getMultiProxyServersConfiguration(s)); + } + + @Override + public String getMultiProxySocketHostHost() { + return getConfig().getMultiProxySocketHostHost(); + } + + @Override + public int getMultiProxySocketHostPort() { + return getConfig().getMultiProxySocketHostPort(); + } + + @Override + public boolean getMultiProxySupportEnabled() { + return getConfig().getMultiProxySupport(); + } + + @Override + public String getMultiProxyUsername() { + return getConfig().getMultiProxyRedisUsername(); + } + + @Override + public File getPluginDataFolder() { + return getDataFolderPlugin(); + } + + @Override + public boolean getPrimaryServer() { + return getConfig().getPrimaryServer(); + } + + @Override + public List getProxyServers() { + return getConfig().getProxyServers(); + } + + @Override + public RedisHandler getRedisHandler() { + return redisHandler; + } + + @Override + public String getVersion() { + return getPluginVersion(); + } + + @Override + public void logInfo(String msg) { + log(msg); + } + + @Override + public void runAsnc(Runnable runnable) { + runAsync(runnable); + } + + @Override + public void setEncryptionHandler(EncryptionHandler encryptionHandler1) { + encryptionHandler = encryptionHandler1; + } + + @Override + public void triggerVote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid) { + vote(player, service, realVote, timeQueue, queueTime, text, uuid); + } + }; + multiProxyHandler.loadMultiProxySupport(); + } + + public abstract void log(String message); + + /** + * Requests a complete player-presence snapshot from one backend server. + * + * @param server configured backend server name + * @return new or already-active request identifier, or null when the server is + * invalid or is inside the snapshot-request cooldown + */ + public UUID requestBackendPresenceSnapshot(String server) { + return requestBackendPresenceSnapshot(server, null); + } + + private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff) { + return requestBackendPresenceSnapshot(server, handoff, System.currentTimeMillis(), false); + } + + private UUID requestBackendPresenceSnapshot(String server, PendingPresenceHandoff handoff, long now, + boolean handoffAlreadyQueued) { + if (method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null + || !isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_SNAPSHOT_REQUEST)) { + return null; + } + long backendStartedAt = backendPlayerPresenceTracker.getBackendStartedAt(server); + UUID backendIncarnationId = backendPlayerPresenceTracker.getBackendIncarnationId(server); + if (backendStartedAt <= 0L || backendIncarnationId == null) { + return null; + } + if (handoff != null && (!server.equalsIgnoreCase(handoff.server) + || !backendIncarnationId.equals(handoff.backendIncarnationId) + || backendStartedAt != handoff.backendStartedAt)) { + return null; + } + if (handoff != null && (handoffAlreadyQueued ? !isPendingPresenceHandoff(handoff, now) + : !queuePendingPresenceHandoff(handoff, now))) { + return null; + } + UUID requestId = handoff == null + ? backendPlayerPresenceTracker.beginSnapshot(server, UUID.randomUUID(), backendIncarnationId, + backendStartedAt, now) + : backendPlayerPresenceTracker.beginSnapshotForDestinationClaim(server, UUID.randomUUID(), + backendIncarnationId, backendStartedAt, handoff.playerUuid, handoff.conflictSequence, now); + boolean created = requestId != null; + if (!created) { + requestId = handoff == null ? backendPlayerPresenceTracker.getPendingSnapshotRequestId(server, now) + : backendPlayerPresenceTracker.getPendingSnapshotRequestIdForDestinationClaim(server, + handoff.playerUuid, handoff.conflictSequence, now); + } + if (requestId == null) { + if (handoff != null && !backendPlayerPresenceTracker.isCurrentDestinationClaim(handoff.playerUuid, + handoff.server, handoff.conflictSequence)) { + discardPendingPresenceHandoff(handoff); + } + // A handoff stays unassigned while the destination is inside its snapshot + // cooldown. Presence maintenance will attach it to the next allowed snapshot. + return null; + } + if (handoff != null) { + assignPendingPresenceHandoff(handoff, requestId, now); + } + if (created) { + JsonEnvelope request = VotingPluginWire.presenceSnapshotRequest(server, requestId, backendIncarnationId, + backendStartedAt, now); + globalMessageProxyHandler.sendMessage(server, 1, request); + } + return requestId; + } + + private boolean queuePendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { + if (!isPresenceHandoffValid(handoff, now)) { + return false; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + PendingPresenceHandoff current = pendingPresenceHandoffs.get(handoff.playerUuid); + if (current != null && current.conflictSequence > handoff.conflictSequence) { + return false; + } + handoff.requestId = null; + pendingPresenceHandoffs.put(handoff.playerUuid, handoff); + return true; + } + } + + private boolean isPendingPresenceHandoff(PendingPresenceHandoff handoff, long now) { + if (!isPresenceHandoffValid(handoff, now)) { + return false; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + return pendingPresenceHandoffs.get(handoff.playerUuid) == handoff; + } + } + + private void assignPendingPresenceHandoff(PendingPresenceHandoff handoff, UUID requestId, long now) { + if (requestId == null || !isPresenceHandoffValid(handoff, now)) { + return; + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + if (pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { + handoff.requestId = requestId; + } + } + } + + private boolean isPresenceHandoffValid(PendingPresenceHandoff handoff, long now) { + return handoff != null && handoff.playerUuid != null && handoff.connectionId != null + && handoff.conflictSequence > 0L + && now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS; + } + + private Set completePendingPresenceHandoffs(UUID requestId, String server, UUID backendIncarnationId, + long backendStartedAt, long now) { + List completed = new ArrayList<>(); + Set completedPlayers = new LinkedHashSet<>(); + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + PendingPresenceHandoff handoff = entry.getValue(); + if (!requestId.equals(handoff.requestId)) { + return false; + } + if (handoff.server.equalsIgnoreCase(server) + && handoff.backendIncarnationId.equals(backendIncarnationId) + && handoff.backendStartedAt == backendStartedAt) { + completed.add(handoff); + } + return true; + }); + } + for (PendingPresenceHandoff handoff : completed) { + PlayerPresence presence = backendPlayerPresenceTracker.getPlayer(handoff.playerUuid).orElse(null); + if (presence != null && presence.getServer().equalsIgnoreCase(handoff.server) + && presence.getConnectionId().equals(handoff.connectionId)) { + login(handoff.playerName, handoff.uuid, handoff.server); + completedPlayers.add(handoff.playerUuid); + } + releaseDestinationClaim(handoff); + } + return completedPlayers; + } + + /** + * Drains voter-keyed cached rewards when a complete recovery snapshot first + * confirms a player on a dedicated voting proxy. Cross-backend handoffs are + * already processed by their token-bound completion path and are excluded to + * avoid a second login callback. + */ + protected void processDedicatedSnapshotLogins(String server, Set handoffPlayers) { + if (!isDedicatedVotingProxyEnabled() || server == null || server.isBlank()) { + return; + } + Set excluded = handoffPlayers == null ? Collections.emptySet() : handoffPlayers; + for (PlayerPresence presence : backendPlayerPresenceTracker.getOnlinePlayers()) { + if (presence.getServer().equalsIgnoreCase(server) && !excluded.contains(presence.getUuid())) { + login(presence.getPlayerName(), presence.getUuid().toString(), presence.getServer()); + } + } + } + + private void discardPendingPresenceHandoff(String uuid) { + try { + UUID playerUuid = UUID.fromString(uuid.trim()); + PendingPresenceHandoff removed; + synchronized (pendingPresenceHandoffs) { + removed = pendingPresenceHandoffs.remove(playerUuid); + } + releaseDestinationClaim(removed); + } catch (Exception ignored) { + // Invalid identities are rejected by the presence tracker. + } + } + + private void discardPendingPresenceHandoff(PendingPresenceHandoff handoff) { + boolean removed = false; + synchronized (pendingPresenceHandoffs) { + if (handoff != null && pendingPresenceHandoffs.get(handoff.playerUuid) == handoff) { + pendingPresenceHandoffs.remove(handoff.playerUuid); + removed = true; + } + } + if (removed) { + releaseDestinationClaim(handoff); + } + } + + private void discardPendingPresenceHandoffs(String server) { + synchronized (pendingPresenceHandoffs) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + if (!entry.getValue().server.equalsIgnoreCase(server)) { + return false; + } + releaseDestinationClaim(entry.getValue()); + return true; + }); + } + } + + private void discardPendingPresenceHandoffs(UUID requestId) { + if (requestId == null) { + return; + } + synchronized (pendingPresenceHandoffs) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + if (!requestId.equals(entry.getValue().requestId)) { + return false; + } + releaseDestinationClaim(entry.getValue()); + return true; + }); + } + } + + private void prunePendingPresenceHandoffs(long now) { + pendingPresenceHandoffs.entrySet().removeIf(entry -> { + PendingPresenceHandoff handoff = entry.getValue(); + if (now >= handoff.createdAt && now - handoff.createdAt <= PRESENCE_HANDOFF_TIMEOUT_MILLIS) { + return false; + } + releaseDestinationClaim(handoff); + return true; + }); + } + + private void releaseDestinationClaim(PendingPresenceHandoff handoff) { + if (handoff != null) { + backendPlayerPresenceTracker.releaseDestinationClaim(handoff.playerUuid, handoff.server, + handoff.conflictSequence); + } + } + + protected void retryPendingPresenceHandoffs(long now) { + List retry = new ArrayList<>(); + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + for (PendingPresenceHandoff handoff : pendingPresenceHandoffs.values()) { + UUID activeRequestId = backendPlayerPresenceTracker.getPendingSnapshotRequestId(handoff.server, now); + if (handoff.requestId != null && !handoff.requestId.equals(activeRequestId)) { + handoff.requestId = null; + } + if (handoff.requestId == null) { + retry.add(handoff); + } + } + } + for (PendingPresenceHandoff handoff : retry) { + requestBackendPresenceSnapshot(handoff.server, handoff, now, true); + } + } + + protected int getPendingPresenceHandoffCount() { + synchronized (pendingPresenceHandoffs) { + return pendingPresenceHandoffs.size(); + } + } + + protected void scheduleBackendPresenceStartupResync() { + ScheduledExecutorService scheduler = getScheduler(); + if (method == null || !method.supportsBackendPresence() || scheduler == null) { + return; + } + scheduler.schedule(this::requestBackendPresenceStartupResync, + PRESENCE_STARTUP_RESYNC_DELAY_SECONDS, TimeUnit.SECONDS); + } + + protected void requestBackendPresenceStartupResync() { + if (!enabled || method == null || !method.supportsBackendPresence() || globalMessageProxyHandler == null) { + return; + } + long requestedAt = System.currentTimeMillis(); + int delay = 1; + for (String server : getAllAvailableServers()) { + if (!isPresenceServerValid(server, VotingPluginWire.SUB_PRESENCE_RESYNC_REQUEST)) { + continue; + } + globalMessageProxyHandler.sendMessage(server, delay++, + VotingPluginWire.presenceResyncRequest(server, UUID.randomUUID(), requestedAt)); + } + } + + private void maintainBackendPresence() { + if (!enabled || method == null || !method.supportsBackendPresence()) { + return; + } + expireBackendPresence(PRESENCE_BACKEND_TIMEOUT_MILLIS); + for (String server : getAllAvailableServers()) { + if (pendingBackendRecoverySnapshots.contains(presenceServerKey(server))) { + requestBackendPresenceSnapshot(server); + } + } + retryPendingPresenceHandoffs(System.currentTimeMillis()); + } + + private String presenceServerKey(String server) { + return server == null ? "" : server.trim().toLowerCase(java.util.Locale.ROOT); + } + + private boolean isPresenceServerValid(String server, String subChannel) { + // The presence protocol's trust boundary is the configured backend set. The + // selected transport must only be accessible to backend servers trusted not to + // impersonate one another. + if (server == null || server.isBlank() || !isServerValid(server)) { + debug("Ignored " + subChannel + " presence envelope for an unconfigured server"); + return false; + } + return true; + } + + private boolean isPresenceGenerationValid(UUID backendIncarnationId, long backendStartedAt, + long presenceTimestamp, String subChannel) { + if (backendIncarnationId == null || backendStartedAt <= 0L || presenceTimestamp < backendStartedAt) { + debug("Ignored " + subChannel + " presence envelope with an invalid backend generation"); + return false; + } + return true; + } + + /** + * Removes presence owned by backends that have stopped reporting heartbeats. + * Scheduling and timeout configuration are intentionally left to dedicated + * proxy mode. + * + * @param timeoutMillis maximum backend silence before expiry + * @return expired backend server names + */ + public Set expireBackendPresence(long timeoutMillis) { + if (method == null || !method.supportsBackendPresence()) { + return Collections.emptySet(); + } + long now = System.currentTimeMillis(); + Set expired = backendPlayerPresenceTracker.expireBackends(now, timeoutMillis); + for (String server : expired) { + discardPendingPresenceHandoffs(server); + // Keep recovery pending while this generation is unavailable. If the same + // backend process resumes, its heartbeat can mark it available again and the + // maintenance task will request a fresh snapshot of players who stayed online. + pendingBackendRecoverySnapshots.add(presenceServerKey(server)); + } + synchronized (pendingPresenceHandoffs) { + prunePendingPresenceHandoffs(now); + } + return expired; + } + + public void login(String playerName, String uuid, String serverName) { + if (!getConfig().getOnlineMode()) { + uuid = getUUID(playerName); + } + + try { + if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { + uuid = UUID.fromString(uuid.trim()).toString(); + } + } catch (Exception ignored) { + // ignore + } + + if (getConfig().getOnlineMode()) { + addNonVotedPlayer(uuid, playerName); + } + if (isPlayerOnlineForVoteRouting(playerName)) { + if (getConfig().getGlobalDataEnabled()) { + if (getGlobalDataHandler().isTimeChangedHappened()) { + getGlobalDataHandler().checkForFinishedTimeChanges(); + } + } + + checkCachedVotes(serverName); + retryPendingOnlineBroadcasts(serverName); + retryPendingTimeBroadcasts(serverName); + checkOnlineVotes(playerName, uuid, serverName); + multiProxyHandler.login(uuid, playerName); + } + } + + private void logInfo(String msg) { + log(msg); + } + + public abstract void logSevere(String message); + + public void onDisable() { + stopControlServices(false); + getVoteCacheHandler().saveVoteCache(); + + if (getProxyMysqlMessenger() != null) { + getProxyMysqlMessenger().shutdown(); + } + + if (getProxyMySQL() != null) { + getProxyMySQL().shutdown(); + } + if (multiProxyHandler != null) { + multiProxyHandler.close(); + } + + if (socketHandler != null) { + socketHandler.closeConnection(); + } + + if (redisHandler != null) { + redisHandler.close(); + } + if (redisPublisherPool != null) { + redisPublisherPool.close(); + redisPublisherPool = null; + } + + bungeeTimeChecker.shutdown(); + + if (getGlobalDataHandler() != null) { + getGlobalDataHandler().shutdown(); + } + + enabled = false; + } + + public void onPluginMessageReceived(DataInputStream in) { + runAsync(() -> { + try { + final String headerSub; + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + headerSub = encryptionHandler.decrypt(in.readUTF()); + } else { + headerSub = in.readUTF(); + } + + int size = in.readInt(); // sanity only + + if (getConfig().getDebug()) { + debug("Received plugin message header=" + headerSub + " size=" + size); + } + + String payload = ""; + if (size > 0) { + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + payload = encryptionHandler.decrypt(in.readUTF()); + } else { + payload = in.readUTF(); + } + } + + JsonEnvelope envelope = JsonEnvelopeCodec.decode(payload); + + if (!headerSub.equalsIgnoreCase(envelope.getSubChannel())) { + if (getConfig().getDebug()) { + warn("PluginMessage subChannel mismatch: header=" + headerSub + " env=" + + envelope.getSubChannel()); + } + return; + } + + globalMessageProxyHandler.onMessage(envelope); + } catch (Exception e) { + e.printStackTrace(); + } + }); + } + + private UUID parseUUIDFromString(String uuidAsString) { + String[] parts = { "0x" + uuidAsString.substring(0, 8), "0x" + uuidAsString.substring(8, 12), + "0x" + uuidAsString.substring(12, 16), "0x" + uuidAsString.substring(16, 20), + "0x" + uuidAsString.substring(20, 32) }; + + long mostSigBits = Long.decode(parts[0]).longValue(); + mostSigBits <<= 16; + mostSigBits |= Long.decode(parts[1]).longValue(); + mostSigBits <<= 16; + mostSigBits |= Long.decode(parts[2]).longValue(); + + long leastSigBits = Long.decode(parts[3]).longValue(); + leastSigBits <<= 48; + leastSigBits |= Long.decode(parts[4]).longValue(); + + return new UUID(mostSigBits, leastSigBits); + } + + public synchronized void processQueue() { + while (getVoteCacheHandler().getTimeChangeQueue().size() > 0) { + VoteTimeQueue vote = getVoteCacheHandler().getTimeChangeQueue().element(); + if (!vote.isProcessed()) { + VoteTotalsSnapshot queuedTotals = vote.getTotals() == null || vote.getTotals().isEmpty() ? null + : VoteTotalsSnapshot.parseStorage(vote.getTotals()); + QueuedVoteResult result = vote(vote.getName(), vote.getService(), true, false, vote.getTime(), queuedTotals, + vote.getUuid(), vote); + if (result == QueuedVoteResult.RETRY) { + scheduleTimeVoteRetry(); + return; + } + if (result == QueuedVoteResult.TERMINAL) { + warn("Removing terminal rollover vote " + vote.getVoteId() + " for " + vote.getName() + "/" + + ServiceSiteValidator.sanitizeForLog(vote.getService())); + } + } + if (!getVoteCacheHandler().removeTimeVote(vote)) { + scheduleTimeVoteRetry(); + return; + } + } + } + + private void scheduleTimeVoteRetry() { + if (timeVoteRetryScheduled || getScheduler() == null) { + return; + } + timeVoteRetryScheduled = true; + try { + getScheduler().schedule(() -> { + synchronized (VotingPluginProxy.this) { + timeVoteRetryScheduled = false; + } + processQueue(); + }, 5, TimeUnit.SECONDS); + } catch (RuntimeException e) { + timeVoteRetryScheduled = false; + debug("Unable to schedule rollover vote retry: " + e.getMessage()); + } + } + + public void reload() { + reloadRuntime(true); + } + + /** Applies a Control-originated configuration reload without stopping its connector or hosted service. */ + public void reloadFromControl() { + reloadRuntime(false); + } + + private void reloadRuntime(boolean restartControlServices) { + method = BungeeMethod.getByName(getConfig().getBungeeMethod()); + if (getMethod() == null) { + method = BungeeMethod.PLUGINMESSAGING; + } + warnUnsupportedDedicatedVotingProxyMode(); + + setCurrentVotePartyVotesRequired( + getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); + loadMultiProxySupport(); + if (restartControlServices) { + startControlServices(); + } + } + + private void warnUnsupportedDedicatedVotingProxyMode() { + if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { + logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, or SOCKETS; PLUGINMESSAGING is disabled for " + + "dedicated-proxy routing. Falling back to normal proxy routing."); + } + } + + public abstract void runAsync(Runnable run); + + /** Platform name used only for the transport-neutral Control discovery contract. */ + public abstract String getProxyPlatform(); + + public abstract void runConsoleCommand(String command); + + public abstract void saveVoteCacheFile(); + + public abstract void reloadCore(boolean mysql); + + /** Strict Control reload path; failures propagate so the caller can restore its backup. */ + public abstract void reloadControlConfiguration() throws Exception; + + public abstract boolean sendPluginMessageData(String server, String channel, byte[] data, boolean queue); + + private static final int PLUGIN_MESSAGE_HARD_LIMIT = 32767; + private static final int PLUGIN_MESSAGE_SOFT_LIMIT = 30000; + + public void sendPluginMessageServer(String server, int delay, JsonEnvelope envelope) { + getScheduler().schedule(() -> sendPluginMessageServerNow(server, envelope), delay * 5L, TimeUnit.MILLISECONDS); + } + + /** + * Sends a plugin-message envelope immediately and reports whether the proxy + * accepted it for delivery. + * + * @param server target backend server + * @param envelope envelope to send + * @return true when the proxy accepted the message for delivery + */ + protected boolean sendPluginMessageServerNow(String server, JsonEnvelope envelope) { + final String subChannel = envelope.getSubChannel(); + final String payload = JsonEnvelopeCodec.encode(envelope); + + final byte[] subChannelBytes = subChannel.getBytes(java.nio.charset.StandardCharsets.UTF_8); + final byte[] payloadBytes = payload.getBytes(java.nio.charset.StandardCharsets.UTF_8); + + // Estimate bytes written: + // - writeUTF adds 2-byte length prefix + UTF-8 bytes + // - writeInt is 4 bytes + int estimatedSize = 2 + subChannelBytes.length + // subChannel UTF (len prefix + bytes) + 4 + // payload length int + 2 + payloadBytes.length; // payload UTF (len prefix + bytes) + + if (estimatedSize > PLUGIN_MESSAGE_SOFT_LIMIT) { + debug("[PluginMessage] Payload nearing limit (" + estimatedSize + " bytes) server=" + server + + " subChannel=" + subChannel + " — consider Redis instead"); + } + + if (estimatedSize > PLUGIN_MESSAGE_HARD_LIMIT) { + debug("[PluginMessage] Payload TOO LARGE (" + estimatedSize + " bytes, max=" + PLUGIN_MESSAGE_HARD_LIMIT + + ") server=" + server + " subChannel=" + subChannel + " — NOT sent"); + return false; + } + + try (ByteArrayOutputStream byteOutStream = new ByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream(byteOutStream)) { + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + out.writeUTF(encryptionHandler.encrypt(subChannel)); + } else { + out.writeUTF(subChannel); + } + + // sanity only: MUST be bytes, not chars + out.writeInt(payloadBytes.length); + + if (getConfig().getPluginMessageEncryption() && encryptionHandler != null) { + out.writeUTF(encryptionHandler.encrypt(payload)); + } else { + out.writeUTF(payload); + } + out.flush(); + + boolean sent = sendPluginMessageData(server, getConfig().getPluginMessageChannel().toLowerCase(), + byteOutStream.toByteArray(), false); + if (getConfig().getDebug()) { + debug((sent ? "Sent" : "Could not send") + " plugin envelope (" + estimatedSize + " bytes) " + server + + " " + subChannel + " " + envelope.getFields()); + } + return sent; + } catch (Exception e) { + e.printStackTrace(); + return false; + } + } + + private DefaultJedisClientConfig buildRedisClientConfig() { + DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder() + .database(getConfig().getRedisDbIndex()).connectionTimeoutMillis(2000).socketTimeoutMillis(2000); + if (getConfig().getRedisUsername() != null && !getConfig().getRedisUsername().isEmpty()) { + config.user(getConfig().getRedisUsername()); + } + if (getConfig().getRedisPassword() != null && !getConfig().getRedisPassword().isEmpty()) { + config.password(getConfig().getRedisPassword()); + } + return config.build(); + } + + public boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope) { + return sendRedisEnvelopeServer(server, envelope, false); + } + + private boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope, boolean useRetryCooldown) { + JedisPool publisherPool = redisPublisherPool; + if (publisherPool == null || (useRetryCooldown && System.currentTimeMillis() < redisPublisherRetryAfter)) { + return false; + } + + try (Jedis jedis = publisherPool.getResource()) { + String channel = getConfig().getRedisPrefix() + "VotingPlugin_" + server; + long subscribers = jedis.publish(channel, JsonEnvelopeCodec.encode(envelope)); + redisPublisherRetryAfter = 0L; + return subscribers > 0; + } catch (Exception e) { + if (useRetryCooldown) { + // Standalone broadcasts remain queued, so their retries can be throttled safely. + redisPublisherRetryAfter = System.currentTimeMillis() + 2000L; + } + debug(e.getMessage()); + return false; + } + } + + public boolean sendMqttEnvelopeServer(String server, JsonEnvelope envelope) { + if (mqttHandler == null) { + return false; + } + try { + mqttHandler.publishEnvelope(getConfig().getMqttPrefix() + "votingplugin/servers/" + server, envelope); + return true; + } catch (Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + return false; + } + } + + public boolean sendSocketEnvelopeServer(String server, JsonEnvelope envelope) { + Map configuration = getConfig().getSpigotServerConfiguration(server); + if (configuration == null) { + return false; + } + String host = configuration.get("Host") instanceof String ? (String) configuration.get("Host") : ""; + int port = configuration.get("Port") instanceof Number ? ((Number) configuration.get("Port")).intValue() : 1298; + if (host.isEmpty()) { + return false; + } + + String payload = JsonEnvelopeCodec.encode(envelope); + String encoded = encryptionHandler != null ? encryptionHandler.encrypt(payload) : payload; + try (Socket socket = new Socket()) { + socket.connect(new InetSocketAddress(host, port), 2000); + try (DataOutputStream output = new DataOutputStream(socket.getOutputStream())) { + output.writeUTF(encoded); + output.flush(); + } + return true; + } catch (Exception e) { + debug(e.getMessage()); + return false; + } + } + + public void sendServerNameMessage() { + for (String s : getAllAvailableServers()) { + sendPluginMessageServer(s, 1, VotingPluginWire.serverName(s)); + } + } + + public void sendVoteParty(String server) { + if (isSomeoneOnlineServerForVoteRouting(server)) { + globalMessageProxyHandler.sendMessage(server, 1, VotingPluginWire.votePartyBungee()); + } + } + + public void setCurrentVotePartyVotes(int amount) { + votePartyVotes = amount; + setVoteCacheVotePartyCurrentVotes(amount); + debug("Current vote party total: " + votePartyVotes); + } + + public abstract void setVoteCacheLastUpdated(); + + public abstract void setVoteCachePrevDay(int day); + + public abstract void setVoteCachePrevMonth(String text); + + public abstract void setVoteCachePrevWeek(int week); + + public abstract void setVoteCacheVoteCacheIgnoreTime(boolean ignore); + + public abstract void setVoteCacheVotePartyCurrentVotes(int votes); + + public abstract void setVoteCacheVotePartyIncreaseVotesRequired(int votes); + + public void status() { + for (String s : getAllAvailableServers()) { + if (!isSomeoneOnlineServerForVoteRouting(s)) { + log("No players on server " + s + " to send test status message, please retest with someone online"); + } else { + log("Sending request for status message on " + s); + globalMessageProxyHandler.sendMessage(s, 1, VotingPluginWire.status(s)); + } + } + } + + private void sendVoteDelayRejected(String player, String uuid, String service, boolean playerOnline, + String playerServer) { + if (!playerOnline || playerServer == null || !getAllAvailableServers().contains(playerServer)) { + debug("Not sending vote delay rejection for " + player + " because the player is offline"); + return; + } + + globalMessageProxyHandler.sendMessage(playerServer, 1, + VotingPluginWire.voteDelayRejected(player, uuid, service, true)); + } + + public String getWaitUntilDelaySiteFromService(String service) { + for (String site : getConfig().getWaitUntilVoteDelaySites()) { + if (getConfig().getWaitUntilVoteDelayService(site).equalsIgnoreCase(service)) { + return site; + } + } + return ""; + } + + private long getLastVotesTime(String uuid, ArrayList cols, String site, String service, String player, + boolean includeTimeChangeQueue) { + long mostRecentTime = 0; + + if (getVoteCacheHandler().hasOnlineVotes(uuid)) { + ArrayList onlineVotes = getVoteCacheHandler().getOnlineVotes(uuid); + for (OfflineBungeeVote vote : onlineVotes) { + if (vote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, vote.getTime()); + } + } + } + + for (String server : getAllAvailableServers()) { + for (OfflineBungeeVote vote : getVoteCacheHandler().getVotes(server)) { + if (vote.getUuid().equals(uuid) && vote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, vote.getTime()); + } + } + } + + if (includeTimeChangeQueue && player != null) { + for (VoteTimeQueue queuedVote : getVoteCacheHandler().getTimeChangeQueue()) { + if (queuedVote.getName().equalsIgnoreCase(player) + && queuedVote.getService().equalsIgnoreCase(service)) { + mostRecentTime = Math.max(mostRecentTime, queuedVote.getTime()); + } + } + } + + for (Column d : cols) { + if (d.getName().equalsIgnoreCase("LastVotes")) { + DataValue value = d.getValue(); + String[] list = value.getString().split("%line%"); + for (String str : list) { + String[] data = str.split("//"); + if (data[0].equalsIgnoreCase(site)) { + mostRecentTime = Math.max(mostRecentTime, Long.valueOf(data[1])); + } + } + } + } + return mostRecentTime; + } + + public boolean checkVoteDelay(String uuid, String service, ArrayList data) { + return checkVoteDelay(uuid, null, service, data, false); + } + + /** + * Checks the configured vote delay, optionally including accepted votes waiting + * for a GlobalData time change to finish. + * + * @param uuid player UUID + * @param player player name used by the time-change queue + * @param service vote service + * @param data current player data + * @param includeTimeChangeQueue whether queued votes reserve their delay slot + * @return true when the vote may be accepted + */ + public boolean checkVoteDelay(String uuid, String player, String service, ArrayList data, + boolean includeTimeChangeQueue) { + String site = getWaitUntilDelaySiteFromService(service); + if (site.isEmpty()) { + debug("No service site set for " + service + ", skipping vote delay check"); + return true; + } + + int voteDelay = getConfig().getWaitUntilVoteDelayVoteDelay(site); + int voteDelayMin = getConfig().getWaitUntilVoteDelayVoteDelayMin(site); + + long lastVote = getLastVotesTime(uuid, data, site, service, player, includeTimeChangeQueue); + if (lastVote == 0) { + debug("No last vote time found for " + uuid + "/" + service + ", skipping vote delay check"); + return true; + } + + try { + LocalDateTime now = getBungeeTimeChecker().getTime(); + LocalDateTime lastVoteTime = LocalDateTime.ofInstant(Instant.ofEpochMilli(lastVote), ZoneId.systemDefault()) + .plusHours(getConfig().getTimeHourOffSet()); + + if (!getConfig().getWaitUntilVoteDelayVoteDelayDaily(site)) { + if (voteDelay == 0 && voteDelayMin == 0) { + debug("Vote delay is 0 for " + site + ", skipping vote delay check"); + return true; + } + + LocalDateTime nextvote = lastVoteTime.plusHours((long) voteDelay).plusMinutes((long) voteDelayMin); + return now.isAfter(nextvote); + } + LocalDateTime resetTime = lastVoteTime.withHour(getConfig().getWaitUntilVoteDelayVoteDelayHour(site)) + .withMinute(0).withSecond(0); + LocalDateTime resetTimeTomorrow = resetTime.plusHours(24); + + if (lastVoteTime.isBefore(resetTime)) { + if (now.isAfter(resetTime)) { + debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); + return true; + } + } else { + if (now.isAfter(resetTimeTomorrow)) { + debug("Vote delay is met for " + uuid + "/" + service + ", vote can be processed"); + return true; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + + debug("Vote delay is not met for " + uuid + "/" + service + ", skipping vote"); + return false; + } + + public synchronized void vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid) { + vote(player, service, realVote, timeQueue, queueTime, text, uuid, null); + } + + private enum QueuedVoteResult { + SUCCESS, RETRY, TERMINAL + } + + private synchronized QueuedVoteResult vote(String player, String service, boolean realVote, boolean timeQueue, long queueTime, + VoteTotalsSnapshot text, String uuid, VoteTimeQueue queuedVote) { + try { + if (!ServiceSiteValidator.isValid(service)) { + warn("Rejected vote with invalid service site '" + ServiceSiteValidator.sanitizeForLog(service) + "'"); + return QueuedVoteResult.TERMINAL; + } + if (!MinecraftUsernameValidator.isValid(player, getConfig().getBedrockPlayerPrefix())) { + warn("Rejected vote with invalid Minecraft username '" + + MinecraftUsernameValidator.sanitizeForLog(player) + "' from service '" + + MinecraftUsernameValidator.sanitizeForLog(service) + "'"); + return QueuedVoteResult.TERMINAL; + } + + UUID voteId = queuedVote == null ? null : queuedVote.getVoteId(); + if (voteId == null) { + voteId = UUID.randomUUID(); + } + + // UUID resolution + if (!getConfig().getOnlineMode()) { + uuid = getUUID(player); + } + + if (uuid == null || uuid.isEmpty()) { + uuid = getUUID(player); + + // Bedrock prefix auto-detect + if (uuid.isEmpty() && !getConfig().getBedrockPlayerPrefix().isEmpty() + && !player.startsWith(getConfig().getBedrockPlayerPrefix())) { + String uuid1 = getUUID(getConfig().getBedrockPlayerPrefix() + player); + if (!uuid1.isEmpty()) { + debug("Detected bedrock player without prefix, adjusting..."); + player = getConfig().getBedrockPlayerPrefix() + player; + uuid = uuid1; + } + } + } + + if (uuid.isEmpty()) { + if (player.startsWith(getConfig().getBedrockPlayerPrefix())) { + log("Ignoring vote since unable to get UUID of bedrock player"); + return QueuedVoteResult.TERMINAL; + } + if (!getConfig().getAllowUnJoined()) { + log("Ignoring vote from " + player + " since player hasn't joined before"); + return QueuedVoteResult.TERMINAL; + } + if (!getConfig().getUUIDLookup()) { + log("Failed to get uuid for " + player); + return QueuedVoteResult.TERMINAL; + } + + debug("Fetching UUID online, since allowunjoined is enabled"); + UUID u = null; + try { + if (getConfig().getOnlineMode()) { + u = fetchUUID(player); + } + } catch (Exception e) { + if (getConfig().getDebug()) { + e.printStackTrace(); + } + } + if (u == null) { + debug("Failed to get uuid for " + player); + return QueuedVoteResult.TERMINAL; + } + uuid = u.toString(); + } + + // Normalize UUID string if possible + try { + if (uuid != null && !uuid.isEmpty() && !uuid.equalsIgnoreCase("null")) { + uuid = UUID.fromString(uuid.trim()).toString(); + } + } catch (Exception ignored) { + // ignore + } + + player = getProperName(uuid, player); + + // Cache online state/server once (IMPORTANT for broadcast logic correctness) + final boolean playerOnline = isPlayerOnlineForVoteRouting(player); + final String playerServer = playerOnline ? getCurrentPlayerServerForVoteRouting(player) : null; + long time = queueTime != 0 ? queueTime + : LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + + Set broadcastTargets = queuedVote == null ? new LinkedHashSet<>() + : new LinkedHashSet<>(queuedVote.getBroadcastTargets()); + Set broadcastForwardedServers = queuedVote == null ? new LinkedHashSet<>() + : new LinkedHashSet<>(queuedVote.getBroadcastForwardedServers()); + boolean proxyBroadcastHandled = queuedVote != null && queuedVote.isProxyBroadcastHandled(); + boolean processesTotals = getConfig().getPrimaryServer() || !getConfig().getMultiProxySupport(); + boolean managesTotals = processesTotals && getConfig().getBungeeManageTotals(); + boolean canValidateStandaloneBroadcast = canForwardStandaloneBroadcast(managesTotals); + ArrayList data = null; + boolean queueForTimeChange = false; + + // A completion callback can wipe totals and replay older queued votes. Run it + // before loading this vote's database snapshot so the calculations below use + // the post-rollover state. + if (getConfig().getGlobalDataEnabled() && getGlobalDataHandler().isTimeChangedHappened()) { + getGlobalDataHandler().checkForFinishedTimeChanges(); + queueForTimeChange = timeQueue && getGlobalDataHandler().isTimeChangedHappened(); + } + + // Validate the vote before any immediate announcement. This keeps duplicate + // votes rejected by the delay check out of the GlobalData rollover queue and + // prevents announcing a vote that will not be processed. + if (managesTotals) { + if (getProxyMySQL() == null) { + logSevere("Mysql is not loaded correctly, stopping vote processing"); + return QueuedVoteResult.RETRY; + } + + if (!getProxyMySQL().containsKeyQuery(uuid)) { + getProxyMySQL().update(uuid, "PlayerName", new DataValueString(player)); + getProxyMySQL().getUuids().add(uuid); + } + + data = getProxyMySQL().getExactQuery(new Column("uuid", new DataValueString(uuid))); + if (!checkVoteDelay(uuid, player, service, data, queuedVote == null)) { + log("Vote delay is not met for " + player + "/" + service + ", skipping vote"); + sendVoteDelayRejected(player, uuid, service, playerOnline, playerServer); + return QueuedVoteResult.TERMINAL; + } + } + + // Forward an accepted offline broadcast before the still-active GlobalData + // change queues the reward/totals work. The queued delivery state prevents + // replaying broadcasts that already reached a backend. + if (queueForTimeChange) { + VoteTotalsSnapshot projectedTotals = managesTotals ? getProjectedRolloverTotals(data, player) : text; + if (canValidateStandaloneBroadcast && proxyBroadcastDecider.usesImmediateForwarding(playerOnline)) { + broadcastTargets.addAll(proxyBroadcastDecider.resolveTargets(false, null)); + proxyBroadcastHandled = true; + } + VoteTimeQueue delayedVote = new VoteTimeQueue(voteId, player, service, time, + proxyBroadcastHandled, broadcastTargets, broadcastForwardedServers, + projectedTotals == null ? "" : projectedTotals.toString(), false, uuid); + if (!getVoteCacheHandler().addTimeVoteToCache(delayedVote)) { + logSevere("Unable to persist queued rollover vote for " + player + "/" + service + + "; skipping proxy broadcast"); + return QueuedVoteResult.RETRY; + } + if (proxyBroadcastHandled) { + for (String target : broadcastTargets) { + Set forwarded = sendProxyBroadcast(Collections.singleton(target), uuid, player, + service, time, projectedTotals == null ? "" : projectedTotals.toString(), false); + if (delayedVote.getBroadcastForwardedServers().addAll(forwarded)) { + broadcastForwardedServers.addAll(forwarded); + persistTimeVoteDelivery(delayedVote); + } + } + } + log("Caching vote from " + player + "/" + service + + " because time change is happening right now"); + return QueuedVoteResult.SUCCESS; + } + + addVoteParty(); + + // Totals processing (primary server OR no multiproxy) + if (processesTotals) { + if (managesTotals) { + int allTimeTotal = getValue(data, "AllTimeTotal", 1); + int monthTotal = getValue(data, "MonthTotal", 1); + + int dateMonthTotal = -1; + if (getConfig().getStoreMonthTotalsWithDate()) { + if (getConfig().getUseMonthDateTotalsAsPrimaryTotal()) { + dateMonthTotal = getValue(data, getMonthTotalsWithDatePath(), 1); + } else { + dateMonthTotal = monthTotal; + } + } + + int weeklyTotal = getValue(data, "WeeklyTotal", 1); + int dailyTotal = getValue(data, "DailyTotal", 1); + int points = getValue(data, "Points", getConfig().getPointsOnVote()); + + int maxVotes = getConfig().getMaxAmountOfVotesPerDay(); + if (maxVotes > 0) { + LocalDateTime cTime = getBungeeTimeChecker().getTime(); + int days = cTime.getDayOfMonth(); + if (monthTotal > days * maxVotes) { + monthTotal = days * maxVotes; + } + } + + if (getConfig().getLimitVotePoints() > 0 && points > getConfig().getLimitVotePoints()) { + points = getConfig().getLimitVotePoints(); + } + + text = new VoteTotalsSnapshot(allTimeTotal, monthTotal, weeklyTotal, dailyTotal, points, + votePartyVotes, currentVotePartyVotesRequired, dateMonthTotal); + + ArrayList update = new ArrayList<>(); + update.add(new Column("AllTimeTotal", new DataValueInt(allTimeTotal))); + update.add(new Column("MonthTotal", new DataValueInt(monthTotal))); + if (getConfig().getStoreMonthTotalsWithDate()) { + update.add(new Column(getMonthTotalsWithDatePath(), new DataValueInt(dateMonthTotal))); + } + update.add(new Column("WeeklyTotal", new DataValueInt(weeklyTotal))); + update.add(new Column("DailyTotal", new DataValueInt(dailyTotal))); + update.add(new Column("Points", new DataValueInt(points))); + + debug("Setting totals " + text.toString() + ", voteId=" + voteId + " for " + player + "/" + + service); + getProxyMySQL().update(uuid, update); + } else { + text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); + } + } + if (text == null) { + text = new VoteTotalsSnapshot(0, 0, 0, 0, 0, votePartyVotes, currentVotePartyVotesRequired, 0); + } + + VoteLogStatus voteStatus = VoteLogStatus.IMMEDIATE; + boolean standaloneProxyBroadcast = canValidateStandaloneBroadcast && (proxyBroadcastHandled + || proxyBroadcastDecider.usesImmediateForwarding(playerOnline)); + Set proxyBroadcastTargets = Collections.emptySet(); + if (standaloneProxyBroadcast) { + // A handled queued broadcast was necessarily sampled while the player was + // offline. Retry only targets that did not previously accept delivery. + proxyBroadcastTargets = proxyBroadcastHandled ? new LinkedHashSet<>(broadcastTargets) + : proxyBroadcastDecider.resolveTargets(false, null); + Set remainingTargets = new LinkedHashSet<>(proxyBroadcastTargets); + remainingTargets.removeAll(broadcastForwardedServers); + broadcastForwardedServers.addAll(sendProxyBroadcast(remainingTargets, uuid, player, service, time, + text == null ? "" : text.toString(), false)); + } + + // =========================== + // Send vote(s) to backend(s) + // =========================== + if (getConfig().getSendVotesToAllServers()) { + for (String s : getAllAvailableServers()) { + + boolean forceCache = getConfig().getWaitForUserOnline() + && (!playerOnline || playerServer == null || !playerServer.equalsIgnoreCase(s)); + + if (forceCache) { + debug("Forcing vote to cache for server " + s); + } + + if ((!isSomeoneOnlineServerForVoteRouting(s) && method.requiresPlayerOnline()) || forceCache) { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addServerVote(s, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, + text.toString(), broadcastForwarded, standaloneProxyBroadcast, + proxyBroadcastTargets, broadcastForwardedServers, false)); + debug("Caching vote for " + player + " on " + service + " for " + s); + } else { + boolean broadcastHere = !broadcastForwardedServers.contains(s); + if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { + Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets + : proxyBroadcastDecider.resolveTargets(playerOnline, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(s, targets); + } + + globalMessageProxyHandler.sendMessage(s, 2, + VotingPluginWire.vote(player, uuid, service, time, true, realVote, text.toString(), + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + } + } + } else { + // Single-server mode: online goes to player server; otherwise queue as "online + // vote" + if (playerOnline && playerServer != null && getAllAvailableServers().contains(playerServer)) { + String server = playerServer; + + boolean broadcastHere = !broadcastForwardedServers.contains(server); + if (broadcastHere && getConfig().getProxyBroadcastEnabled()) { + Set targets = standaloneProxyBroadcast ? proxyBroadcastTargets + : proxyBroadcastDecider.resolveTargets(true, playerServer); + broadcastHere = proxyBroadcastDecider.shouldBroadcast(server, targets); + } + + globalMessageProxyHandler.sendMessage(server, 1, + VotingPluginWire.voteOnline(player, uuid, service, time, true, realVote, text.toString(), + voteId, getConfig().getBungeeManageTotals(), broadcastHere, 1, 1)); + + if (canValidateStandaloneBroadcast && getConfig().getProxyBroadcastEnabled() + && !standaloneProxyBroadcast) { + Set targets = proxyBroadcastDecider.resolveTargets(true, playerServer); + + int bDelay = 2; + for (String targetServer : targets) { + // avoid double-broadcast on the same server that already got the voteOnline + if (targetServer.equalsIgnoreCase(server)) { + continue; + } + if (getConfig().getBlockedServers().contains(targetServer)) { + continue; + } + + globalMessageProxyHandler.sendMessage(targetServer, bDelay, + VotingPluginWire.voteBroadcast(uuid, player, service, time, + text == null ? "" : text.toString(), true)); + bDelay++; + } + } + + // multiproxy: envelope-only clear vote + if (getConfig().getMultiProxySupport() && getConfig().getMultiProxyOneGlobalReward()) { + multiProxyHandler.sendClearVote(uuid, player); + } + } else { + voteStatus = VoteLogStatus.CACHED; + boolean broadcastForwarded = standaloneProxyBroadcast + && broadcastForwardedServers.containsAll(proxyBroadcastTargets); + getVoteCacheHandler().addOnlineVote(uuid, + new OfflineBungeeVote(voteId, player, uuid, service, time, realVote, text.toString(), + broadcastForwarded, standaloneProxyBroadcast, proxyBroadcastTargets, + broadcastForwardedServers, false)); + debug("Caching online vote for " + player + " on " + service); + } + + int delay = 2; + for (String s : getAllAvailableServers()) { + globalMessageProxyHandler.sendMessage(s, delay + 1, VotingPluginWire.voteUpdate(uuid, + votePartyVotes, currentVotePartyVotesRequired, service, time, text.toString())); + delay += 2; + } + } + + // Vote logging + if (voteLogMysqlTable != null && getConfig().getVoteLoggingEnabled()) { + voteLogMysqlTable.logVote(voteId, voteStatus, service, uuid, player, time, + getVoteCacheHandler().getProxyCachedTotal(uuid)); + } + + // =========================== + // Multiproxy forwarding + // =========================== + if (getConfig().getMultiProxySupport() && getConfig().getPrimaryServer()) { + if (!getConfig().getMultiProxyOneGlobalReward()) { + debug("Sending global proxy vote envelope"); + multiProxyHandler.sendMultiProxyEnvelope(VotingPluginWire.vote(player, uuid, service, time, false, + realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); + } else { + // Only send to other proxies if the player DID NOT already receive reward on a + // backend + boolean shouldSend = true; + if (playerOnline && playerServer != null) { + if (!getConfig().getBlockedServers().contains(playerServer)) { + shouldSend = false; + } + } + + if (shouldSend) { + debug("Sending global proxy voteonline envelope"); + multiProxyHandler + .sendMultiProxyEnvelope(VotingPluginWire.voteOnline(player, uuid, service, time, false, + realVote, text == null ? "" : text.toString(), voteId, false, false, 1, 1)); + } else { + debug("Not sending global proxy message for voteonline, player already got reward"); + } + } + } + if (queuedVote != null) { + queuedVote.setProcessed(true); + if (!getVoteCacheHandler().updateTimeVote(queuedVote)) { + warn("Unable to persist completed rollover vote " + queuedVote.getVoteId() + + "; attempting durable removal immediately"); + } + } + return QueuedVoteResult.SUCCESS; + } catch (Exception e) { + e.printStackTrace(); + return QueuedVoteResult.RETRY; + } + } + + private static final class PendingPresenceHandoff { + private UUID requestId; + private final UUID playerUuid; + private final String playerName; + private final String uuid; + private final String server; + private final UUID connectionId; + private final UUID backendIncarnationId; + private final long backendStartedAt; + private final long conflictSequence; + private final long createdAt; + + private PendingPresenceHandoff(String playerName, String uuid, String server, UUID connectionId, + UUID backendIncarnationId, long backendStartedAt, long conflictSequence, long createdAt) { + this.playerUuid = parsePlayerUuid(uuid); + this.playerName = playerName; + this.uuid = uuid; + this.server = server; + this.connectionId = connectionId; + this.backendIncarnationId = backendIncarnationId; + this.backendStartedAt = backendStartedAt; + this.conflictSequence = conflictSequence; + this.createdAt = createdAt; + } + + private static UUID parsePlayerUuid(String uuid) { + try { + return UUID.fromString(uuid.trim()); + } catch (Exception ignored) { + return null; + } + } + } + + public abstract void warn(String message); + + public abstract ScheduledExecutorService getScheduler(); +} From 1f1a8f12728f09a6a881a153597cec93e08c1539 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 24 Aug 2026 20:50:49 -0600 Subject: [PATCH 07/73] Honor negotiated Control capabilities --- .../proxy/control/ControlConnector.java | 6 ++- .../ProxyRoutingConfigurationService.java | 6 +-- .../proxy/control/ControlConnectorTest.java | 52 ++++++++++++++++--- .../ProxyRoutingConfigurationServiceTest.java | 6 ++- docs/control-connector.md | 3 +- 5 files changed, 59 insertions(+), 14 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index d1ea86aa1d..6bec21b4ae 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -65,6 +65,7 @@ public final class ControlConnector implements AutoCloseable { private final AtomicBoolean inFlight = new AtomicBoolean(); private volatile boolean closed; private volatile boolean registered; + private volatile boolean configurationAccepted; private volatile int failures; private volatile long snapshotSequence; private volatile ScheduledFuture scheduled; @@ -184,6 +185,7 @@ void cycle() { activeRequest = presence; return presence.thenCompose(responseBody -> { handlePresenceResponse(responseBody); + if (!configurationAccepted) return CompletableFuture.completedFuture(null); CompletableFuture claim = transport.send(claimRequest()); activeRequest = claim; return claim.thenCompose(this::handleClaimResponse); @@ -210,16 +212,18 @@ private void handlePrimaryResponse(Response response, boolean registration) { } requireSuccess(response); JsonObject body = parseObject(response.body); + JsonObject node = body.getAsJsonObject("node"); + JsonArray accepted = node == null ? null : node.getAsJsonArray("acceptedCapabilities"); if (registration) { JsonElement protocol = body.getAsJsonObject("identity").get("protocolVersion"); if (protocol == null || protocol.getAsInt() != PROTOCOL_VERSION) { throw new ProtocolException(); } - JsonArray accepted = body.getAsJsonObject("node").getAsJsonArray("acceptedCapabilities"); if (accepted == null || !contains(accepted, "presence.snapshot")) { throw new ProtocolException(); } } + configurationAccepted = accepted != null && contains(accepted, CONFIGURATION_CAPABILITY); } private void handlePresenceResponse(Response response) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java index 9c9014d845..264995ac93 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.util.HashSet; -import java.util.Locale; import java.util.Set; import com.bencodez.votingplugin.proxy.VotingPluginProxy; @@ -36,10 +35,9 @@ public ProxyRoutingConfiguration read() { } public void validate(ProxyRoutingConfiguration proposal) { - Set available = new HashSet<>(); - platform.configuredServers().forEach(server -> available.add(server.toLowerCase(Locale.ROOT))); + Set available = new HashSet<>(platform.configuredServers()); for (String server : proposal.blockedServers()) { - if (!available.contains(server.toLowerCase(Locale.ROOT))) { + if (!available.contains(server)) { throw new IllegalArgumentException("unknown backend: " + server); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 5e0dbf919a..472b0b0eb6 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -43,7 +43,7 @@ class ControlConnectorTest { @Test void registrationHeartbeatAndPresenceAreWireCompatibleAndReplacementSnapshotsAdvance() { connector.cycle(); assertEquals(Status.CONNECTED, connector.status()); - assertEquals(3, transport.requests.size()); + assertEquals(2, transport.requests.size()); Request registration = transport.requests.get(0); assertEquals("POST", registration.method()); assertEquals("/api/v1/nodes/register", registration.path()); @@ -59,8 +59,8 @@ class ControlConnectorTest { .get("backendId").getAsString()); connector.cycle(); - assertEquals("/api/v1/nodes/proxy-a/heartbeat", transport.requests.get(3).path()); - JsonObject secondSnapshot = JsonParser.parseString(transport.requests.get(4).body()).getAsJsonObject(); + assertEquals("/api/v1/nodes/proxy-a/heartbeat", transport.requests.get(2).path()); + JsonObject secondSnapshot = JsonParser.parseString(transport.requests.get(3).body()).getAsJsonObject(); assertEquals(1, secondSnapshot.get("sequence").getAsLong()); } @@ -89,8 +89,36 @@ class ControlConnectorTest { connector.cycle(); assertEquals(Status.UNAVAILABLE, connector.status()); connector.cycle(); - assertEquals("POST", transport.requests.get(4).method()); - assertEquals("/api/v1/nodes/register", transport.requests.get(4).path()); + assertEquals("POST", transport.requests.get(3).method()); + assertEquals("/api/v1/nodes/register", transport.requests.get(3).path()); + } + + @Test void skipsOperationClaimsWhenConfigurationCapabilityWasNotAccepted() { + connector.close(); + ProxyRoutingConfigurationService service = new ProxyRoutingConfigurationService(new NoOpPlatform()); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L, service); + transport.acceptConfiguration = false; + + connector.cycle(); + + assertEquals(Status.CONNECTED, connector.status()); + assertEquals(2, transport.requests.size()); + assertTrue(transport.requests.stream().noneMatch(request -> request.path().endsWith("/operations"))); + } + + @Test void claimsOperationsWhenConfigurationCapabilityWasAccepted() { + connector.close(); + ProxyRoutingConfigurationService service = new ProxyRoutingConfigurationService(new NoOpPlatform()); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L, service); + transport.acceptConfiguration = true; + + connector.cycle(); + + assertEquals(Status.CONNECTED, connector.status()); + assertEquals(3, transport.requests.size()); + assertTrue(transport.requests.get(2).path().endsWith("/operations")); } @Test void slowTransportDoesNotBlockCallerAndShutdownCancelsInFlightRequest() { @@ -147,6 +175,7 @@ private static final class FakeTransport implements Transport { private Response nextPrimary; private CompletableFuture stalled; private RuntimeException synchronousFailure; + private boolean acceptConfiguration; @Override public CompletableFuture send(Request request) { @@ -171,12 +200,23 @@ public CompletableFuture send(Request request) { return CompletableFuture.completedFuture(response); } if ("/api/v1/nodes/register".equals(request.path())) { + String capabilities = acceptConfiguration + ? "[\"presence.snapshot\",\"config.proxy-routing.v1\"]" : "[\"presence.snapshot\"]"; return CompletableFuture.completedFuture(new Response(201, - "{\"identity\":{\"protocolVersion\":1},\"node\":{\"acceptedCapabilities\":[\"presence.snapshot\"]}}")); + "{\"identity\":{\"protocolVersion\":1},\"node\":{\"acceptedCapabilities\":" + + capabilities + "}}")); } return CompletableFuture.completedFuture(new Response(200, "{\"node\":{}}")); } return CompletableFuture.completedFuture(new Response(200, "{\"applied\":true,\"node\":{}}")); } } + + private static final class NoOpPlatform implements ProxyRoutingConfigurationService.Platform { + @Override public ProxyRoutingConfiguration read() { return new ProxyRoutingConfiguration(false, List.of()); } + @Override public java.util.Set configuredServers() { return java.util.Set.of(); } + @Override public void persist(ProxyRoutingConfiguration proposal) { } + @Override public void rollback() { } + @Override public void reload() { } + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java index 9037f5e076..30ebfdb40f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java @@ -16,8 +16,10 @@ void validatesBlockedServersAgainstCompleteConfiguredSet() { FakePlatform platform = new FakePlatform(); platform.configuredServers = Set.of("Lobby"); - assertDoesNotThrow(() -> new ProxyRoutingConfigurationService(platform) - .validate(new ProxyRoutingConfiguration(true, List.of("lobby")))); + ProxyRoutingConfigurationService service = new ProxyRoutingConfigurationService(platform); + assertDoesNotThrow(() -> service.validate(new ProxyRoutingConfiguration(true, List.of("Lobby")))); + assertThrows(IllegalArgumentException.class, + () -> service.validate(new ProxyRoutingConfiguration(true, List.of("lobby")))); } @Test diff --git a/docs/control-connector.md b/docs/control-connector.md index 27859c74b7..24bed6d11a 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -130,7 +130,8 @@ invent an online state from a server address or arbitrary wall-clock timestamp. The connector requires `presence.snapshot` and advertises `config.proxy-routing.v1` when its typed adapter is available. The latter permits reads and preview/apply of only `SendVotesToAllServers` and `BlockedServers`. Every preview validates -blocked names against that proxy's configured backends and returns a deterministic SHA-256 revision. Apply rejects stale +blocked names as exact, case-preserving matches against that proxy's configured backends and returns a deterministic +SHA-256 revision. Apply rejects stale revisions, writes a local `.control-backup`, requires atomic activation of a staged YAML file, and soft-reloads the proxy. A reload failure triggers immediate backup restoration and another reload attempt; Control receives the per-node reload/rollback result. Task results are cached by operation ID so a leased retry cannot apply the same change twice. From c35e5bc470068104f2dc46ffb3e80dfe360fa544 Mon Sep 17 00:00:00 2001 From: Ben Date: Mon, 24 Aug 2026 20:57:42 -0600 Subject: [PATCH 08/73] Preserve negotiated Control capabilities --- .../bencodez/votingplugin/proxy/control/ControlConnector.java | 4 +++- .../votingplugin/proxy/control/ControlConnectorTest.java | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 6bec21b4ae..534ad0906c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -223,7 +223,9 @@ private void handlePrimaryResponse(Response response, boolean registration) { throw new ProtocolException(); } } - configurationAccepted = accepted != null && contains(accepted, CONFIGURATION_CAPABILITY); + if (accepted != null) { + configurationAccepted = contains(accepted, CONFIGURATION_CAPABILITY); + } } private void handlePresenceResponse(Response response) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 472b0b0eb6..95a6e57cad 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -119,6 +119,10 @@ class ControlConnectorTest { assertEquals(Status.CONNECTED, connector.status()); assertEquals(3, transport.requests.size()); assertTrue(transport.requests.get(2).path().endsWith("/operations")); + + connector.cycle(); + assertEquals(6, transport.requests.size()); + assertTrue(transport.requests.get(5).path().endsWith("/operations")); } @Test void slowTransportDoesNotBlockCallerAndShutdownCancelsInFlightRequest() { From 9d26134ec54bd2040ba7dfbeda75ec17e9a4f36a Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 08:43:54 -0600 Subject: [PATCH 09/73] Add Bukkit configuration Control connector --- .../votingplugin/VotingPluginMain.java | 19 + .../control/BackendConfigurationService.java | 266 +++++++++++++ .../control/BackendControlConnector.java | 363 ++++++++++++++++++ .../votingplugin/proxy/VotingPluginProxy.java | 33 +- .../control/ProxyRoutingConfiguration.java | 2 +- VotingPlugin/src/main/resources/Config.yml | 18 + .../BackendConfigurationServiceTest.java | 70 ++++ .../ProxyRoutingConfigurationTest.java | 7 +- docs/control-connector.md | 42 +- 9 files changed, 796 insertions(+), 24 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 875c6a4fec..310187af24 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -77,6 +77,7 @@ import com.bencodez.votingplugin.config.ShopFile; import com.bencodez.votingplugin.config.SpecialRewardsConfig; import com.bencodez.votingplugin.cooldown.CoolDownCheck; +import com.bencodez.votingplugin.control.BackendControlConnector; import com.bencodez.votingplugin.data.ServerData; import com.bencodez.votingplugin.discord.DiscordHandler; import com.bencodez.votingplugin.listeners.BlockBreak; @@ -267,6 +268,8 @@ public class VotingPluginMain extends AdvancedCorePlugin { @Getter private DiscordHandler discordHandler; + private BackendControlConnector backendControlConnector; + public void addDirectlyDefinedRewards(DirectlyDefinedReward directlyDefinedReward) { getRewardHandler().addDirectlyDefined(directlyDefinedReward); } @@ -1577,6 +1580,18 @@ public void run() { }, 10); } + startBackendControlConnector(); + + } + + private void startBackendControlConnector() { + try { + backendControlConnector = BackendControlConnector.create(this); + if (backendControlConnector != null) backendControlConnector.start(); + } catch (Exception e) { + backendControlConnector = null; + getLogger().warning("[Control] Bukkit configuration connector was not started: " + e.getMessage()); + } } private void migrateVoteBroadcast(Config configFile) { @@ -1680,6 +1695,10 @@ public void onFinish() { */ @Override public void onUnLoad() { + if (backendControlConnector != null) { + backendControlConnector.close(); + backendControlConnector = null; + } if (getBackendProxyHandler() != null) { try { getBackendProxyHandler().close(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java new file mode 100644 index 0000000000..3ff1d2ee07 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -0,0 +1,266 @@ +package com.bencodez.votingplugin.control; + +import java.io.IOException; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.bukkit.configuration.ConfigurationSection; +import org.bukkit.configuration.InvalidConfigurationException; +import org.bukkit.configuration.file.YamlConfiguration; + +/** Safe, revisioned access to every user-facing VotingPlugin YAML configuration file. */ +public final class BackendConfigurationService { + public static final String REDACTED = "__VOTINGPLUGIN_CONTROL_REDACTED__"; + public static final int MAX_CONTENT_BYTES = 512 * 1024; + private static final Set TOP_LEVEL = Set.of("Config.yml", "VoteSites.yml", "SpecialRewards.yml", + "GUI.yml", "Shop.yml", "BungeeSettings.yml"); + + private final Path dataDirectory; + private final ReloadAction reload; + + public BackendConfigurationService(Path dataDirectory, ReloadAction reload) { + this.dataDirectory = dataDirectory.toAbsolutePath().normalize(); + this.reload = reload; + } + + public Document read(String fileName) throws IOException { + Path path = resolve(fileName); + String raw = readRaw(path, false); + YamlConfiguration yaml = parse(raw); + return new Document(fileName, mask(yaml), revision(raw)); + } + + public Preview preview(String fileName, String proposedContent) throws IOException { + Path path = resolve(fileName); + String current = readRaw(path, false); + String resolved = resolveSecrets(parse(proposedContent), parse(current)).saveToString(); + ensureBounded(resolved); + return new Preview(fileName, resolved, revision(current), changes(parse(current), parse(resolved))); + } + + public ApplyResult apply(String fileName, String proposedContent, String expectedRevision) throws IOException { + Path target = resolve(fileName); + String current = readRaw(target, false); + if (expectedRevision == null || !revision(current).equals(expectedRevision)) throw new StaleRevisionException(); + Preview preview = preview(fileName, proposedContent); + Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); + Path staging = Files.createTempFile(target.getParent(), ".control-", ".yml"); + boolean installed = false; + try { + Files.copy(target, backup, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + Files.writeString(staging, preview.resolvedContent(), StandardCharsets.UTF_8, + StandardOpenOption.TRUNCATE_EXISTING); + move(staging, target); + installed = true; + reload.run(); + String applied = readRaw(target, false); + return new ApplyResult(new Document(fileName, mask(parse(applied)), revision(applied)), + preview.changes(), false); + } catch (Exception failure) { + boolean rolledBack = false; + if (installed) { + try { + Files.copy(backup, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + reload.run(); + rolledBack = true; + } catch (Exception rollbackFailure) { + failure.addSuppressed(rollbackFailure); + } + } + throw new ApplyFailureException(rolledBack, failure); + } finally { + Files.deleteIfExists(staging); + } + } + + public QuickPreview previewQuickSetup(String preset, Map options) throws IOException { + QuickProposal proposal = quickProposal(preset, options); + String current = readRaw(resolve(proposal.fileName()), false); + return new QuickPreview(proposal, revision(current), changes(parse(current), parse(proposal.content()))); + } + + public ApplyResult applyQuickSetup(String preset, Map options, String expectedRevision) + throws IOException { + QuickProposal proposal = quickProposal(preset, options); + return apply(proposal.fileName(), proposal.content(), expectedRevision); + } + + private QuickProposal quickProposal(String preset, Map options) throws IOException { + if ("standalone".equals(preset) || "proxy-backend".equals(preset)) { + String fileName = "BungeeSettings.yml"; + YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); + boolean proxy = "proxy-backend".equals(preset); + yaml.set("UseBungeecord", proxy); + if (proxy) { + String server = option(options, "server", "[A-Za-z0-9][A-Za-z0-9._-]{0,63}"); + yaml.set("Server", server); + yaml.set("BungeeMethod", options.getOrDefault("method", "PLUGINMESSAGING")); + } + return new QuickProposal(fileName, yaml.saveToString()); + } + if ("vote-site".equals(preset)) { + String fileName = "VoteSites.yml"; + YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); + String name = option(options, "name", "[A-Za-z0-9_-]{1,64}"); + String root = "VoteSites." + name; + yaml.set(root + ".Enabled", true); + yaml.set(root + ".Name", options.getOrDefault("displayName", name)); + yaml.set(root + ".Priority", boundedInteger(options.getOrDefault("priority", "5"), 0, 100)); + yaml.set(root + ".Hidden", false); + yaml.set(root + ".ServiceSite", option(options, "serviceSite", ".{1,200}")); + yaml.set(root + ".VoteURL", option(options, "voteUrl", ".{1,500}")); + yaml.set(root + ".VoteDelay", options.getOrDefault("voteDelay", "24h")); + yaml.set(root + ".DisplayItem.Material", options.getOrDefault("material", "DIAMOND")); + yaml.set(root + ".DisplayItem.Amount", 1); + return new QuickProposal(fileName, yaml.saveToString()); + } + throw new IllegalArgumentException("quick setup preset is unsupported"); + } + + private Path resolve(String fileName) { + if (fileName == null || (!TOP_LEVEL.contains(fileName) + && !fileName.matches("VoteSites/[A-Za-z0-9._-]{1,100}\\.yml"))) { + throw new IllegalArgumentException("configuration file is not managed"); + } + Path target = dataDirectory.resolve(fileName).normalize(); + if (!target.startsWith(dataDirectory)) throw new IllegalArgumentException("configuration path escapes data folder"); + return target; + } + + private static String readRaw(Path path, boolean allowMissing) throws IOException { + if (!Files.exists(path) && allowMissing) return ""; + if (!Files.isRegularFile(path) || Files.size(path) > MAX_CONTENT_BYTES) { + throw new IOException("configuration file is missing or too large"); + } + byte[] bytes = Files.readAllBytes(path); + try { + return StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(java.nio.ByteBuffer.wrap(bytes)).toString(); + } catch (CharacterCodingException e) { + throw new IOException("configuration file is not valid UTF-8", e); + } + } + + private static YamlConfiguration parse(String content) { + if (content == null) throw new IllegalArgumentException("configuration content is required"); + ensureBounded(content); + YamlConfiguration yaml = new YamlConfiguration(); + try { + yaml.loadFromString(content); + } catch (InvalidConfigurationException e) { + throw new IllegalArgumentException("configuration is not valid YAML"); + } + return yaml; + } + + private static void ensureBounded(String content) { + if (content == null || content.indexOf('\0') >= 0 + || content.getBytes(StandardCharsets.UTF_8).length > MAX_CONTENT_BYTES) { + throw new IllegalArgumentException("configuration content exceeds the 512 KiB limit"); + } + } + + private static String mask(YamlConfiguration yaml) { + for (String path : new ArrayList<>(yaml.getKeys(true))) { + if (!(yaml.get(path) instanceof ConfigurationSection) && secret(path)) yaml.set(path, REDACTED); + } + return yaml.saveToString(); + } + + private static YamlConfiguration resolveSecrets(YamlConfiguration proposal, YamlConfiguration current) { + for (String path : new ArrayList<>(proposal.getKeys(true))) { + if (secret(path) && REDACTED.equals(proposal.getString(path))) proposal.set(path, current.get(path)); + } + return proposal; + } + + private static boolean secret(String path) { + String key = path.substring(path.lastIndexOf('.') + 1).replace("-", "").replace("_", "") + .toLowerCase(Locale.ROOT); + return key.contains("password") || key.contains("secret") || key.equals("token") + || key.equals("apikey") || key.equals("authorization") || key.equals("webhookurl") || key.equals("url") + && path.toLowerCase(Locale.ROOT).contains("webhook"); + } + + private static List changes(YamlConfiguration current, YamlConfiguration proposal) { + Set paths = new LinkedHashSet<>(current.getKeys(true)); + paths.addAll(proposal.getKeys(true)); + List changes = new ArrayList<>(); + for (String path : paths) { + Object before = current.get(path); + Object after = proposal.get(path); + if (before instanceof ConfigurationSection || after instanceof ConfigurationSection) continue; + if (!java.util.Objects.equals(before, after)) { + changes.add((secret(path) ? "changed secret " : "changed ") + path); + if (changes.size() == 19) { + changes.add("additional changes omitted"); + break; + } + } + } + return List.copyOf(changes); + } + + private static String revision(String content) { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") + .digest(content.getBytes(StandardCharsets.UTF_8))); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 is unavailable", e); + } + } + + private static void move(Path source, Path target) throws IOException { + try { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); + } + } + + private static String option(Map options, String name, String pattern) { + String value = options == null ? null : options.get(name); + if (value == null || !value.matches(pattern)) throw new IllegalArgumentException("quick setup option " + name + " is invalid"); + return value; + } + + private static int boundedInteger(String value, int minimum, int maximum) { + try { + int parsed = Integer.parseInt(value); + if (parsed < minimum || parsed > maximum) throw new NumberFormatException(); + return parsed; + } catch (NumberFormatException e) { + throw new IllegalArgumentException("quick setup number is invalid"); + } + } + + public record Document(String fileName, String content, String revision) { } + public record Preview(String fileName, String resolvedContent, String revision, List changes) { } + public record ApplyResult(Document document, List changes, boolean rolledBack) { } + public record QuickProposal(String fileName, String content) { } + public record QuickPreview(QuickProposal proposal, String revision, List changes) { } + + @FunctionalInterface public interface ReloadAction { void run() throws Exception; } + @SuppressWarnings("serial") public static final class StaleRevisionException extends RuntimeException { } + @SuppressWarnings("serial") public static final class ApplyFailureException extends IOException { + private final boolean rolledBack; + private ApplyFailureException(boolean rolledBack, Throwable cause) { super(cause); this.rolledBack = rolledBack; } + public boolean rolledBack() { return rolledBack; } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java new file mode 100644 index 0000000000..78f6f00b69 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -0,0 +1,363 @@ +package com.bencodez.votingplugin.control; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.regex.Pattern; + +import org.bukkit.configuration.ConfigurationSection; + +import com.bencodez.votingplugin.VotingPluginMain; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** Optional outbound Bukkit node connector for full configuration-file and quick-setup control. */ +public final class BackendControlConnector implements AutoCloseable { + private static final int PROTOCOL_VERSION = 1; + private static final int MAX_RESPONSE_BYTES = 768 * 1024; + private static final Pattern NODE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,63}"); + private static final Set CAPABILITIES = Set.of("config.files.v1", "config.quick-setup.v1"); + + private final VotingPluginMain plugin; + private final Settings settings; + private final String credential; + private final ScheduledExecutorService executor; + private final HttpClient http; + private final BackendConfigurationService configurations; + private final UUID sessionId = UUID.randomUUID(); + private final Map completed = new LinkedHashMap<>(); + private final AtomicBoolean running = new AtomicBoolean(); + private volatile boolean closed; + private volatile boolean registered; + private volatile boolean operationsAccepted; + private volatile int failures; + private volatile ScheduledFuture scheduled; + + private BackendControlConnector(VotingPluginMain plugin, Settings settings, String credential) { + this.plugin = plugin; + this.settings = settings; + this.credential = credential; + ThreadFactory factory = runnable -> { + Thread thread = new Thread(runnable, "votingplugin-control-backend"); + thread.setDaemon(true); + return thread; + }; + executor = Executors.newSingleThreadScheduledExecutor(factory); + http = HttpClient.newBuilder().connectTimeout(Duration.ofMillis(settings.connectTimeoutMillis())) + .followRedirects(HttpClient.Redirect.NEVER).build(); + configurations = new BackendConfigurationService(plugin.getDataFolder().toPath(), () -> + plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + plugin.reload(); + return null; + }).get(30, TimeUnit.SECONDS)); + } + + public static BackendControlConnector create(VotingPluginMain plugin) throws IOException { + ConfigurationSection control = plugin.getConfigFile().getData().getConfigurationSection("Control.Backend"); + if (control == null || !control.getBoolean("Enabled", false)) return null; + String nodeId = control.getString("NodeId", "").trim(); + if (!NODE_ID.matcher(nodeId).matches()) throw new IllegalArgumentException("Control.Backend.NodeId is invalid"); + String endpointValue = control.getString("Endpoint", "").trim(); + URI endpoint = URI.create(endpointValue); + if (!Set.of("http", "https").contains(endpoint.getScheme()) || endpoint.getHost() == null + || endpoint.getUserInfo() != null || endpoint.getQuery() != null || endpoint.getFragment() != null + || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { + throw new IllegalArgumentException("Control.Backend.Endpoint must be an HTTP(S) origin"); + } + Path root = plugin.getDataFolder().toPath().toAbsolutePath().normalize(); + Path credentialPath = root.resolve(control.getString("CredentialFile", "control-credential.txt")).normalize(); + if (!credentialPath.startsWith(root) || !Files.isRegularFile(credentialPath) || Files.size(credentialPath) > 512) { + throw new IOException("Control backend credential file is missing or invalid"); + } + String credential = Files.readString(credentialPath, StandardCharsets.UTF_8).trim(); + if (credential.isEmpty() || credential.length() > 512 || credential.indexOf('\r') >= 0 || credential.indexOf('\n') >= 0) { + throw new IOException("Control backend credential file is missing or invalid"); + } + Settings settings = new Settings(nodeId, endpoint, + bounded(control.getInt("HeartbeatSeconds", 30), 10, 300, "HeartbeatSeconds"), + bounded(control.getInt("ConnectTimeoutMillis", 3000), 500, 30000, "ConnectTimeoutMillis"), + bounded(control.getInt("RequestTimeoutMillis", 10000), 500, 30000, "RequestTimeoutMillis")); + return new BackendControlConnector(plugin, settings, credential); + } + + public void start() { schedule(0); } + + private void schedule(long delayMillis) { + if (!closed) scheduled = executor.schedule(this::cycle, delayMillis, TimeUnit.MILLISECONDS); + } + + private void cycle() { + if (closed || !running.compareAndSet(false, true)) return; + try { + JsonObject primary = registered ? heartbeat() : register(); + JsonObject node = primary.has("node") ? primary.getAsJsonObject("node") : null; + if (!registered) { + JsonObject identity = primary.has("identity") ? primary.getAsJsonObject("identity") : null; + if (identity == null || identity.get("protocolVersion").getAsInt() != PROTOCOL_VERSION) { + throw new ConnectorException("protocol mismatch"); + } + } + if (node != null && node.has("acceptedCapabilities")) { + operationsAccepted = containsAll(node.getAsJsonArray("acceptedCapabilities"), CAPABILITIES); + } + registered = true; + if (operationsAccepted) claimAndExecute(); + if (failures > 0) plugin.getLogger().info("[Control] Bukkit configuration connector recovered"); + failures = 0; + } catch (Exception failure) { + failures = Math.min(30, failures + 1); + if (failures == 1 || failures % 10 == 0) { + plugin.getLogger().warning("[Control] Bukkit configuration connector unavailable; VotingPlugin remains active"); + } + } finally { + running.set(false); + long delay = failures == 0 ? TimeUnit.SECONDS.toMillis(settings.heartbeatSeconds()) + : Math.min(TimeUnit.MINUTES.toMillis(5), 1000L << Math.min(failures - 1, 8)); + schedule(delay); + } + } + + private JsonObject register() throws Exception { + JsonObject body = sessionBody(); + body.addProperty("nodeId", settings.nodeId()); + body.addProperty("displayName", settings.nodeId()); + body.addProperty("platform", "BUKKIT"); + body.addProperty("pluginVersion", plugin.getDescription().getVersion()); + addCapabilities(body); + return requireObject(send("POST", "/api/v1/nodes/register", body), 200, 201); + } + + private JsonObject heartbeat() throws Exception { + JsonObject body = sessionBody(); + addCapabilities(body); + Response response = send("PUT", "/api/v1/nodes/" + settings.nodeId() + "/heartbeat", body); + if (response.status() == 404) { + registered = false; + return register(); + } + return requireObject(response, 200); + } + + private void claimAndExecute() throws Exception { + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + Response response = send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations", body); + if (response.status() == 204) return; + JsonObject task = requireObject(response, 200); + UUID operationId = UUID.fromString(string(task, "operationId")); + TaskResult result; + synchronized (completed) { + result = completed.get(operationId); + } + if (result == null) { + result = execute(task); + synchronized (completed) { + completed.put(operationId, result); + while (completed.size() > 128) completed.remove(completed.keySet().iterator().next()); + } + } + JsonObject resultBody = result.json(sessionId); + requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", + resultBody), 200); + } + + private TaskResult execute(JsonObject task) { + try { + String type = string(task, "type"); + JsonObject configuration = task.getAsJsonObject("configuration"); + String domain = string(configuration, "domain"); + if ("file".equals(domain)) return executeFile(type, configuration, task); + if ("quick-setup".equals(domain)) return executeQuick(type, configuration, task); + return TaskResult.failure("UNSUPPORTED_TASK", "Configuration domain is unsupported"); + } catch (BackendConfigurationService.StaleRevisionException e) { + return TaskResult.failure("STALE_REVISION", "Configuration changed after preview"); + } catch (BackendConfigurationService.ApplyFailureException e) { + return TaskResult.failure("RELOAD_FAILED", "Reload failed after persistence", e.rolledBack()); + } catch (IllegalArgumentException e) { + return TaskResult.failure("VALIDATION_ERROR", e.getMessage()); + } catch (Exception e) { + return TaskResult.failure("APPLY_FAILED", "Configuration operation failed"); + } + } + + private TaskResult executeFile(String type, JsonObject configuration, JsonObject task) throws IOException { + String fileName = string(configuration, "fileName"); + if ("READ".equals(type)) { + BackendConfigurationService.Document document = configurations.read(fileName); + return TaskResult.file(document, List.of(), false, false); + } + String content = string(configuration, "content"); + if ("PREVIEW".equals(type)) { + BackendConfigurationService.Preview preview = configurations.preview(fileName, content); + BackendConfigurationService.Document current = configurations.read(fileName); + return TaskResult.file(current, preview.changes(), false, false); + } + if ("APPLY".equals(type)) { + BackendConfigurationService.ApplyResult applied = configurations.apply(fileName, content, + string(task, "expectedRevision")); + return TaskResult.file(applied.document(), applied.changes(), true, applied.rolledBack()); + } + return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); + } + + private TaskResult executeQuick(String type, JsonObject configuration, JsonObject task) throws IOException { + if ("READ".equals(type)) return TaskResult.failure("UNSUPPORTED_TASK", "Quick setups cannot be read"); + String preset = string(configuration, "preset"); + Map options = options(configuration.getAsJsonObject("options")); + if ("PREVIEW".equals(type)) { + BackendConfigurationService.QuickPreview preview = configurations.previewQuickSetup(preset, options); + return TaskResult.quick(preset, options, preview.revision(), preview.changes(), false); + } + if ("APPLY".equals(type)) { + BackendConfigurationService.ApplyResult applied = configurations.applyQuickSetup(preset, options, + string(task, "expectedRevision")); + return TaskResult.quick(preset, options, applied.document().revision(), applied.changes(), true); + } + return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); + } + + private Response send(String method, String path, JsonObject body) throws Exception { + HttpRequest request = HttpRequest.newBuilder(settings.endpoint().resolve(path)) + .timeout(Duration.ofMillis(settings.requestTimeoutMillis())).header("Content-Type", "application/json") + .header("Authorization", "Bearer " + credential) + .method(method, HttpRequest.BodyPublishers.ofString(body.toString(), StandardCharsets.UTF_8)).build(); + HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); + try (InputStream input = response.body()) { + byte[] bytes = input.readNBytes(MAX_RESPONSE_BYTES + 1); + if (bytes.length > MAX_RESPONSE_BYTES) throw new ConnectorException("response too large"); + return new Response(response.statusCode(), new String(bytes, StandardCharsets.UTF_8)); + } + } + + private static JsonObject requireObject(Response response, int... statuses) { + boolean accepted = false; + for (int status : statuses) if (response.status() == status) accepted = true; + if (!accepted) throw new ConnectorException("unexpected response"); + JsonElement element = JsonParser.parseString(response.body()); + if (!element.isJsonObject()) throw new ConnectorException("malformed response"); + return element.getAsJsonObject(); + } + + private JsonObject sessionBody() { + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + body.addProperty("protocolVersion", PROTOCOL_VERSION); + return body; + } + + private static void addCapabilities(JsonObject body) { + JsonArray capabilities = new JsonArray(); + CAPABILITIES.stream().sorted().forEach(capabilities::add); + body.add("capabilities", capabilities); + JsonArray required = new JsonArray(); + required.add("config.files.v1"); + body.add("requiredCapabilities", required); + } + + private static boolean containsAll(JsonArray accepted, Set expected) { + Set values = new java.util.HashSet<>(); + if (accepted != null) accepted.forEach(value -> values.add(value.getAsString())); + return values.containsAll(expected); + } + + private static String string(JsonObject object, String name) { + if (object == null || !object.has(name) || object.get(name).isJsonNull() + || !object.get(name).isJsonPrimitive()) throw new IllegalArgumentException(name + " is required"); + return object.get(name).getAsString(); + } + + private static Map options(JsonObject object) { + Map values = new LinkedHashMap<>(); + if (object != null) object.entrySet().forEach(entry -> values.put(entry.getKey(), entry.getValue().getAsString())); + return Map.copyOf(values); + } + + private static int bounded(int value, int min, int max, String name) { + if (value < min || value > max) throw new IllegalArgumentException("Control.Backend." + name + " is invalid"); + return value; + } + + @Override + public void close() { + closed = true; + ScheduledFuture current = scheduled; + if (current != null) current.cancel(false); + executor.shutdownNow(); + } + + private record Settings(String nodeId, URI endpoint, int heartbeatSeconds, int connectTimeoutMillis, + int requestTimeoutMillis) { } + private record Response(int status, String body) { } + + private record TaskResult(boolean success, String code, String message, String revision, + JsonObject configuration, List changes, boolean reloaded, boolean rolledBack) { + private JsonObject json(UUID sessionId) { + JsonObject body = new JsonObject(); + body.addProperty("sessionId", sessionId.toString()); + body.addProperty("success", success); + body.addProperty("code", code); + body.addProperty("message", message); + if (revision != null) body.addProperty("revision", revision); + if (configuration != null) body.add("configuration", configuration); + JsonArray listed = new JsonArray(); + changes.forEach(listed::add); + body.add("changes", listed); + body.addProperty("reloaded", reloaded); + body.addProperty("rolledBack", rolledBack); + return body; + } + + private static TaskResult file(BackendConfigurationService.Document document, List changes, + boolean reloaded, boolean rolledBack) { + JsonObject config = new JsonObject(); + config.addProperty("domain", "file"); + config.addProperty("fileName", document.fileName()); + config.addProperty("content", document.content()); + return new TaskResult(true, "OK", "Operation completed", document.revision(), config, + List.copyOf(changes), reloaded, rolledBack); + } + + private static TaskResult quick(String preset, Map options, String revision, + List changes, boolean reloaded) { + JsonObject config = new JsonObject(); + config.addProperty("domain", "quick-setup"); + config.addProperty("preset", preset); + JsonObject values = new JsonObject(); + options.forEach(values::addProperty); + config.add("options", values); + return new TaskResult(true, "OK", "Operation completed", revision, config, List.copyOf(changes), + reloaded, false); + } + + private static TaskResult failure(String code, String message) { return failure(code, message, false); } + private static TaskResult failure(String code, String message, boolean rolledBack) { + return new TaskResult(false, code, message == null ? "Operation failed" : message, null, null, + List.of(), false, rolledBack); + } + } + + @SuppressWarnings("serial") private static final class ConnectorException extends RuntimeException { + private ConnectorException(String message) { super(message); } + } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index e51b4e4f66..2e71ed9c11 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1320,22 +1320,7 @@ public void onReceiveEnvelope(JsonEnvelope envelope) { } }); - clientHandles = new HashMap<>(); - List l = getConfig().getBlockedServers(); - for (String s : getConfig().getSpigotServers()) { - if (!l.contains(s)) { - Map d = getConfig().getSpigotServerConfiguration(s); - String host = ""; - if (d.containsKey("Host")) { - host = (String) d.get("Host"); - } - int port = 1298; - if (d.containsKey("Port")) { - port = (int) d.get("Port"); - } - clientHandles.put(s, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); - } - } + rebuildSocketClients(); } else if (method.equals(BungeeMethod.REDIS)) { redisHandler = new RedisHandler(getConfig().getRedisHost(), getConfig().getRedisPort(), getConfig().getRedisUsername(), getConfig().getRedisPassword(), getConfig().getRedisDbIndex()) { @@ -2476,6 +2461,9 @@ private void reloadRuntime(boolean restartControlServices) { method = BungeeMethod.PLUGINMESSAGING; } warnUnsupportedDedicatedVotingProxyMode(); + if (!restartControlServices && method == BungeeMethod.SOCKETS) { + rebuildSocketClients(); + } setCurrentVotePartyVotesRequired( getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); @@ -2485,6 +2473,19 @@ private void reloadRuntime(boolean restartControlServices) { } } + private void rebuildSocketClients() { + HashMap rebuilt = new HashMap<>(); + List blocked = getConfig().getBlockedServers(); + for (String server : getConfig().getSpigotServers()) { + if (blocked.contains(server)) continue; + Map data = getConfig().getSpigotServerConfiguration(server); + String host = data.containsKey("Host") ? (String) data.get("Host") : ""; + int port = data.containsKey("Port") ? (int) data.get("Port") : 1298; + rebuilt.put(server, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); + } + clientHandles = rebuilt; + } + private void warnUnsupportedDedicatedVotingProxyMode() { if (getConfig().getDedicatedVotingProxy() && (method == null || !method.supportsBackendPresence())) { logSevere("DedicatedVotingProxy requires MYSQL, REDIS, MQTT, or SOCKETS; PLUGINMESSAGING is disabled for " diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfiguration.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfiguration.java index 87d4758897..1033c6edee 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfiguration.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfiguration.java @@ -26,7 +26,7 @@ public record ProxyRoutingConfiguration(boolean sendVotesToAllServers, List value.toLowerCase(Locale.ROOT)).sorted().toList()); + .sorted().toList()); try { return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256") .digest(canonical.getBytes(StandardCharsets.UTF_8))); diff --git a/VotingPlugin/src/main/resources/Config.yml b/VotingPlugin/src/main/resources/Config.yml index 3f6f7b2eca..b39337e1f8 100644 --- a/VotingPlugin/src/main/resources/Config.yml +++ b/VotingPlugin/src/main/resources/Config.yml @@ -1158,6 +1158,24 @@ Webhooks: Secret: "change-me" ########################################### +# VotingPlugin Control backend connector +# Optional outbound connection for full local WebUI configuration control. +# Create a node credential with Control's `enroll` owner command and place it +# in the VotingPlugin data folder. The credential is never stored in this YAML. +########################################### + +Control: + Backend: + Enabled: false + # Must be unique for every Bukkit/Paper backend in the network. + NodeId: '' + Endpoint: 'http://127.0.0.1:8080' + CredentialFile: 'control-credential.txt' + HeartbeatSeconds: 30 + ConnectTimeoutMillis: 3000 + RequestTimeoutMillis: 10000 + +########################################### # Additional / Advanced Settings # Rarely needed options for special setups ########################################### diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java new file mode 100644 index 0000000000..7fc837beb0 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -0,0 +1,70 @@ +package com.bencodez.votingplugin.control; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.concurrent.atomic.AtomicInteger; + +import org.bukkit.configuration.file.YamlConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class BackendConfigurationServiceTest { + @TempDir Path directory; + + @Test void masksSecretsPreservesThemAndAppliesARevisionedReload() throws Exception { + Path config = directory.resolve("Config.yml"); + Files.writeString(config, "Database:\n Password: keep-me\nFeature: false\n"); + AtomicInteger reloads = new AtomicInteger(); + BackendConfigurationService service = new BackendConfigurationService(directory, reloads::incrementAndGet); + + BackendConfigurationService.Document read = service.read("Config.yml"); + assertFalse(read.content().contains("keep-me")); + assertTrue(read.content().contains(BackendConfigurationService.REDACTED)); + String proposal = read.content().replace("Feature: false", "Feature: true"); + BackendConfigurationService.Preview preview = service.preview("Config.yml", proposal); + assertTrue(preview.changes().contains("changed Feature")); + + BackendConfigurationService.ApplyResult result = service.apply("Config.yml", proposal, read.revision()); + assertEquals(1, reloads.get()); + YamlConfiguration applied = new YamlConfiguration(); + applied.loadFromString(Files.readString(config)); + assertEquals("keep-me", applied.getString("Database.Password")); + assertTrue(applied.getBoolean("Feature")); + assertFalse(result.document().content().contains("keep-me")); + assertTrue(Files.isRegularFile(directory.resolve("Config.yml.control-backup"))); + } + + @Test void rejectsStaleInvalidAndUnmanagedWrites() throws Exception { + Files.writeString(directory.resolve("Config.yml"), "Feature: false\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + String revision = service.read("Config.yml").revision(); + Files.writeString(directory.resolve("Config.yml"), "Feature: manual\n"); + assertThrows(BackendConfigurationService.StaleRevisionException.class, + () -> service.apply("Config.yml", "Feature: true\n", revision)); + assertThrows(IllegalArgumentException.class, () -> service.preview("../server.properties", "x: y")); + assertThrows(IllegalArgumentException.class, () -> service.preview("Config.yml", "bad: [yaml")); + } + + @Test void quickSetupsProduceReviewableBackendAndVoteSiteChanges() throws Exception { + Files.writeString(directory.resolve("BungeeSettings.yml"), "UseBungeecord: false\nServer: PleaseSet\n"); + Files.writeString(directory.resolve("VoteSites.yml"), "VoteSites: {}\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + + BackendConfigurationService.QuickPreview proxy = service.previewQuickSetup("proxy-backend", + Map.of("server", "lobby", "method", "PLUGINMESSAGING")); + assertTrue(proxy.proposal().content().contains("UseBungeecord: true")); + assertTrue(proxy.proposal().content().contains("Server: lobby")); + + BackendConfigurationService.QuickPreview site = service.previewQuickSetup("vote-site", Map.of( + "name", "PMC", "serviceSite", "PlanetMinecraft.com", "voteUrl", "https://example.com/vote", + "voteDelay", "24h")); + assertTrue(site.proposal().content().contains("PMC:")); + assertTrue(site.proposal().content().contains("PlanetMinecraft.com")); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationTest.java index 0a80cd3868..e4ac9f4da8 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationTest.java @@ -1,16 +1,19 @@ package com.bencodez.votingplugin.proxy.control; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import java.util.List; import org.junit.jupiter.api.Test; class ProxyRoutingConfigurationTest { - @Test void revisionIsStableAcrossBlockedServerCaseAndOrder() { + @Test void revisionIgnoresOrderButPreservesBehaviorallySignificantCase() { ProxyRoutingConfiguration first = new ProxyRoutingConfiguration(true, List.of("Lobby", "survival")); - ProxyRoutingConfiguration second = new ProxyRoutingConfiguration(true, List.of("SURVIVAL", "lobby")); + ProxyRoutingConfiguration second = new ProxyRoutingConfiguration(true, List.of("survival", "Lobby")); assertEquals(first.revision(), second.revision()); + assertNotEquals(first.revision(), + new ProxyRoutingConfiguration(true, List.of("lobby", "survival")).revision()); } @Test void rejectsDuplicateBlankAndOversizedServerLists() { diff --git a/docs/control-connector.md b/docs/control-connector.md index 24bed6d11a..142146e07f 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -2,10 +2,10 @@ This development milestone adds an optional connector from the BungeeCord and Velocity VotingPlugin proxy JAR to the separate [VotingPlugin Control](https://github.com/BenCodez/VotingPlugin-Control) application. It discovers each -proxy and its eligible backend server names. Control includes the local WebUI. VotingPlugin may now explicitly opt in to +proxy and its eligible backend server names. Enrolled Bukkit nodes can also connect outbound for full configuration-file +and quick-setup control. Control includes the local WebUI. VotingPlugin may now explicitly opt in to provisioning and supervising that application as a separate child JVM; it is never loaded into the proxy classloader. The -connector does not expose backend addresses, secrets, or unrestricted configuration and does not connect directly from -Bukkit backend nodes. +connector does not expose backend addresses. Configuration secrets are masked on every read and never enter audit records. ## Failure isolation @@ -116,6 +116,28 @@ Timing bounds are intentional: Do not raise these by patching around validation; correct the topology or connectivity problem instead. +### Bukkit full-configuration enrollment + +Each backend is separately opt-in and separately enrolled so one node credential cannot impersonate another. Put its +credential in that backend's VotingPlugin data folder and configure `Config.yml`: + +```yaml +Control: + Backend: + Enabled: true + NodeId: 'backend-lobby' + Endpoint: 'http://127.0.0.1:8080' + CredentialFile: 'control-credential.txt' + HeartbeatSeconds: 30 + ConnectTimeoutMillis: 3000 + RequestTimeoutMillis: 10000 +``` + +The Bukkit connector owns one daemon worker and performs no Control I/O on the server thread. It negotiates +`config.files.v1` and `config.quick-setup.v1`, then polls the same outbound operation queue as proxies. File apply schedules +the VotingPlugin reload on the Bukkit thread and waits only on the connector worker. Control failure never blocks votes, +joins, commands, or plugin shutdown. + ## Discovery semantics Both platforms use the same connector implementation and protocol version `1`. Each proxy process creates a new session @@ -136,7 +158,17 @@ revisions, writes a local `.control-backup`, requires atomic activation of a sta proxy. A reload failure triggers immediate backup restoration and another reload attempt; Control receives the per-node reload/rollback result. Task results are cached by operation ID so a leased retry cannot apply the same change twice. -The WebUI never sends raw YAML. An admin must select capable online proxies, preview the typed change, and confirm the +For enrolled Bukkit nodes, the WebUI can read and edit all six user-facing files: `Config.yml`, `VoteSites.yml`, +`SpecialRewards.yml`, `GUI.yml`, `Shop.yml`, and `BungeeSettings.yml`. YAML is parsed before preview; each apply uses an +exact SHA-256 revision, stages and atomically installs the file, retains `.control-backup`, reloads, and restores the backup +if reload fails. Returned YAML is normalized and masks password/secret/token/API-key/authorization/webhook-secret paths +with `__VOTINGPLUGIN_CONTROL_REDACTED__`; leaving the marker unchanged preserves the local value. A replacement secret may +be submitted through the authenticated preview, but is never returned or audited. + +Quick setups currently cover standalone backend mode, proxy-connected backend mode with an explicit server identity, and +adding/updating a vote site. They use the same preview, revision, approval, backup, reload, and rollback path—not a bypass. + +An admin must select capable online nodes, preview the change, and confirm the single-use approval generated for that exact successful preview. Nodes claim work over their existing outbound connection, so no inbound listener is added to VotingPlugin. Authentication failures, unavailable required capabilities, and protocol mismatches back off only the connector. Current redacted diagnostic status is one of `DISABLED`, `STARTING`, @@ -154,6 +186,6 @@ default. - Backend listed with unknown presence: this is expected when the selected existing VotingPlugin transport does not provide backend-presence observations. -Unrestricted configuration, manual rollback, topology persistence, cloud relay, diagnostics downloads, signed remote +Arbitrary console commands, manual rollback, topology persistence, cloud relay, diagnostics downloads, signed remote release manifests, and remote support remain later milestones. Hosted downloads currently require the SHA-256 trust pin to be supplied in local configuration; VotingPlugin never trusts a remotely fetched checksum by itself. From 26784b0c6deaff2bfe7a7caf76cd18238a68e5b1 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 10:24:11 -0600 Subject: [PATCH 10/73] Add plugin-aware quick configuration setups --- .../votingplugin/VotingPluginMain.java | 7 ++ .../control/BackendConfigurationService.java | 81 +++++++++++++++++-- .../control/BackendControlConnector.java | 53 ++++++++---- .../BackendConfigurationServiceTest.java | 20 +++++ docs/control-connector.md | 13 ++- 5 files changed, 150 insertions(+), 24 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 310187af24..b513a23aee 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1594,6 +1594,13 @@ private void startBackendControlConnector() { } } + /** Refreshes the optional Control connector after its Config.yml settings were applied. */ + public synchronized void restartBackendControlConnector() { + if (backendControlConnector != null) backendControlConnector.close(); + backendControlConnector = null; + startBackendControlConnector(); + } + private void migrateVoteBroadcast(Config configFile) { ConfigurationSection cfg = configFile.getData(); // If new section exists, do nothing diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 3ff1d2ee07..93e2d58643 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -49,29 +49,31 @@ public Document read(String fileName) throws IOException { public Preview preview(String fileName, String proposedContent) throws IOException { Path path = resolve(fileName); String current = readRaw(path, false); - String resolved = resolveSecrets(parse(proposedContent), parse(current)).saveToString(); - ensureBounded(resolved); - return new Preview(fileName, resolved, revision(current), changes(parse(current), parse(resolved))); + return preview(fileName, proposedContent, current); } public ApplyResult apply(String fileName, String proposedContent, String expectedRevision) throws IOException { Path target = resolve(fileName); String current = readRaw(target, false); if (expectedRevision == null || !revision(current).equals(expectedRevision)) throw new StaleRevisionException(); - Preview preview = preview(fileName, proposedContent); + Preview preview = preview(fileName, proposedContent, current); Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); Path staging = Files.createTempFile(target.getParent(), ".control-", ".yml"); boolean installed = false; try { - Files.copy(target, backup, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); Files.writeString(staging, preview.resolvedContent(), StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING); + Files.writeString(backup, current, StandardCharsets.UTF_8, StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); move(staging, target); installed = true; reload.run(); String applied = readRaw(target, false); return new ApplyResult(new Document(fileName, mask(parse(applied)), revision(applied)), preview.changes(), false); + } catch (StaleRevisionException stale) { + throw stale; } catch (Exception failure) { boolean rolledBack = false; if (installed) { @@ -89,6 +91,13 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte } } + private Preview preview(String fileName, String proposedContent, String current) { + YamlConfiguration currentYaml = parse(current); + String resolved = resolveSecrets(parse(proposedContent), currentYaml).saveToString(); + ensureBounded(resolved); + return new Preview(fileName, resolved, revision(current), changes(currentYaml, parse(resolved))); + } + public QuickPreview previewQuickSetup(String preset, Map options) throws IOException { QuickProposal proposal = quickProposal(preset, options); String current = readRaw(resolve(proposal.fileName()), false); @@ -130,6 +139,51 @@ private QuickProposal quickProposal(String preset, Map options) yaml.set(root + ".DisplayItem.Amount", 1); return new QuickProposal(fileName, yaml.saveToString()); } + if ("easy-reward".equals(preset)) { + String fileName = "VoteSites.yml"; + YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); + String scope = options == null ? "site" : options.getOrDefault("scope", "site"); + String root; + if ("every-site".equals(scope)) { + root = "EverySiteReward"; + } else if ("site".equals(scope)) { + root = "VoteSites." + option(options, "name", "[A-Za-z0-9_-]{1,64}") + ".Rewards"; + } else { + throw new IllegalArgumentException("quick setup option scope is invalid"); + } + String command = optional(options, "command", 500); + String message = optional(options, "message", 500); + if (command.isBlank() && message.isBlank()) { + throw new IllegalArgumentException("easy reward requires a command or player message"); + } + if (!command.isBlank()) yaml.set(root + ".Commands", List.of(command)); + if (!message.isBlank()) yaml.set(root + ".Messages.Player", message); + return new QuickProposal(fileName, yaml.saveToString()); + } + if ("common-settings".equals(preset)) { + String fileName = "Config.yml"; + YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); + yaml.set("ProcessRewards", booleanOption(options, "processRewards")); + yaml.set("AutoCreateVoteSites", booleanOption(options, "autoCreateVoteSites")); + yaml.set("ExtraAllSitesCheck", booleanOption(options, "extraAllSitesCheck")); + yaml.set("CountFakeVotes", booleanOption(options, "countFakeVotes")); + yaml.set("DisableNoServiceSiteMessage", booleanOption(options, "disableNoServiceSiteMessage")); + yaml.set("DisableUpdateChecking", booleanOption(options, "disableUpdateChecking")); + return new QuickProposal(fileName, yaml.saveToString()); + } + if ("vote-party".equals(preset)) { + String fileName = "SpecialRewards.yml"; + YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); + yaml.set("VoteParty.Enabled", true); + yaml.set("VoteParty.VotesRequired", boundedInteger(option(options, "votesRequired", "[0-9]{1,6}"), 1, 100000)); + yaml.set("VoteParty.GiveAllPlayers", booleanOption(options, "giveAllPlayers")); + yaml.set("VoteParty.GiveOnlinePlayersOnly", booleanOption(options, "onlineOnly")); + String command = optional(options, "command", 500); + String broadcast = optional(options, "broadcast", 500); + if (!command.isBlank()) yaml.set("VoteParty.Rewards.Commands", List.of(command)); + if (!broadcast.isBlank()) yaml.set("VoteParty.Broadcast", broadcast); + return new QuickProposal(fileName, yaml.saveToString()); + } throw new IllegalArgumentException("quick setup preset is unsupported"); } @@ -240,6 +294,23 @@ private static String option(Map options, String name, String pa return value; } + private static String optional(Map options, String name, int maximum) { + String value = options == null ? "" : options.getOrDefault(name, "").trim(); + if (value.length() > maximum || value.indexOf('\0') >= 0 || value.indexOf('\r') >= 0 + || value.indexOf('\n') >= 0) { + throw new IllegalArgumentException("quick setup option " + name + " is invalid"); + } + return value; + } + + private static boolean booleanOption(Map options, String name) { + String value = options == null ? null : options.get(name); + if (!"true".equals(value) && !"false".equals(value)) { + throw new IllegalArgumentException("quick setup option " + name + " is invalid"); + } + return Boolean.parseBoolean(value); + } + private static int boundedInteger(String value, int minimum, int maximum) { try { int parsed = Integer.parseInt(value); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 78f6f00b69..bb8bb6ce6a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -51,6 +51,7 @@ public final class BackendControlConnector implements AutoCloseable { private volatile boolean closed; private volatile boolean registered; private volatile boolean operationsAccepted; + private volatile boolean quickSetupsAccepted; private volatile int failures; private volatile ScheduledFuture scheduled; @@ -119,7 +120,9 @@ private void cycle() { } } if (node != null && node.has("acceptedCapabilities")) { - operationsAccepted = containsAll(node.getAsJsonArray("acceptedCapabilities"), CAPABILITIES); + JsonArray accepted = node.getAsJsonArray("acceptedCapabilities"); + operationsAccepted = contains(accepted, "config.files.v1"); + quickSetupsAccepted = contains(accepted, "config.quick-setup.v1"); } registered = true; if (operationsAccepted) claimAndExecute(); @@ -144,6 +147,11 @@ private JsonObject register() throws Exception { body.addProperty("displayName", settings.nodeId()); body.addProperty("platform", "BUKKIT"); body.addProperty("pluginVersion", plugin.getDescription().getVersion()); + JsonArray detectedPlugins = new JsonArray(); + java.util.Arrays.stream(plugin.getServer().getPluginManager().getPlugins()) + .map(installed -> installed.getDescription().getName()).filter(name -> name != null && !name.isBlank()) + .distinct().sorted(String.CASE_INSENSITIVE_ORDER).limit(128).forEach(detectedPlugins::add); + body.add("detectedPlugins", detectedPlugins); addCapabilities(body); return requireObject(send("POST", "/api/v1/nodes/register", body), 200, 201); } @@ -180,6 +188,10 @@ private void claimAndExecute() throws Exception { JsonObject resultBody = result.json(sessionId); requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", resultBody), 200); + if (result.restartConnector()) { + synchronized (completed) { completed.remove(operationId); } + plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); + } } private TaskResult execute(JsonObject task) { @@ -188,7 +200,10 @@ private TaskResult execute(JsonObject task) { JsonObject configuration = task.getAsJsonObject("configuration"); String domain = string(configuration, "domain"); if ("file".equals(domain)) return executeFile(type, configuration, task); - if ("quick-setup".equals(domain)) return executeQuick(type, configuration, task); + if ("quick-setup".equals(domain)) { + if (!quickSetupsAccepted) return TaskResult.failure("UNSUPPORTED_TASK", "Quick setups were not negotiated"); + return executeQuick(type, configuration, task); + } return TaskResult.failure("UNSUPPORTED_TASK", "Configuration domain is unsupported"); } catch (BackendConfigurationService.StaleRevisionException e) { return TaskResult.failure("STALE_REVISION", "Configuration changed after preview"); @@ -205,18 +220,20 @@ private TaskResult executeFile(String type, JsonObject configuration, JsonObject String fileName = string(configuration, "fileName"); if ("READ".equals(type)) { BackendConfigurationService.Document document = configurations.read(fileName); - return TaskResult.file(document, List.of(), false, false); + return TaskResult.file(document, List.of(), false, false, false); } String content = string(configuration, "content"); if ("PREVIEW".equals(type)) { BackendConfigurationService.Preview preview = configurations.preview(fileName, content); BackendConfigurationService.Document current = configurations.read(fileName); - return TaskResult.file(current, preview.changes(), false, false); + if (!preview.revision().equals(current.revision())) throw new BackendConfigurationService.StaleRevisionException(); + return TaskResult.file(current, preview.changes(), false, false, false); } if ("APPLY".equals(type)) { BackendConfigurationService.ApplyResult applied = configurations.apply(fileName, content, string(task, "expectedRevision")); - return TaskResult.file(applied.document(), applied.changes(), true, applied.rolledBack()); + return TaskResult.file(applied.document(), applied.changes(), true, applied.rolledBack(), + "Config.yml".equals(fileName)); } return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); } @@ -232,7 +249,8 @@ private TaskResult executeQuick(String type, JsonObject configuration, JsonObjec if ("APPLY".equals(type)) { BackendConfigurationService.ApplyResult applied = configurations.applyQuickSetup(preset, options, string(task, "expectedRevision")); - return TaskResult.quick(preset, options, applied.document().revision(), applied.changes(), true); + return TaskResult.quick(preset, options, applied.document().revision(), applied.changes(), true, + "Config.yml".equals(applied.document().fileName())); } return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); } @@ -275,10 +293,10 @@ private static void addCapabilities(JsonObject body) { body.add("requiredCapabilities", required); } - private static boolean containsAll(JsonArray accepted, Set expected) { - Set values = new java.util.HashSet<>(); - if (accepted != null) accepted.forEach(value -> values.add(value.getAsString())); - return values.containsAll(expected); + private static boolean contains(JsonArray accepted, String expected) { + if (accepted == null) return false; + for (JsonElement value : accepted) if (value.isJsonPrimitive() && expected.equals(value.getAsString())) return true; + return false; } private static String string(JsonObject object, String name) { @@ -311,7 +329,8 @@ private record Settings(String nodeId, URI endpoint, int heartbeatSeconds, int c private record Response(int status, String body) { } private record TaskResult(boolean success, String code, String message, String revision, - JsonObject configuration, List changes, boolean reloaded, boolean rolledBack) { + JsonObject configuration, List changes, boolean reloaded, boolean rolledBack, + boolean restartConnector) { private JsonObject json(UUID sessionId) { JsonObject body = new JsonObject(); body.addProperty("sessionId", sessionId.toString()); @@ -329,17 +348,21 @@ private JsonObject json(UUID sessionId) { } private static TaskResult file(BackendConfigurationService.Document document, List changes, - boolean reloaded, boolean rolledBack) { + boolean reloaded, boolean rolledBack, boolean restartConnector) { JsonObject config = new JsonObject(); config.addProperty("domain", "file"); config.addProperty("fileName", document.fileName()); config.addProperty("content", document.content()); return new TaskResult(true, "OK", "Operation completed", document.revision(), config, - List.copyOf(changes), reloaded, rolledBack); + List.copyOf(changes), reloaded, rolledBack, restartConnector); } private static TaskResult quick(String preset, Map options, String revision, List changes, boolean reloaded) { + return quick(preset, options, revision, changes, reloaded, false); + } + private static TaskResult quick(String preset, Map options, String revision, + List changes, boolean reloaded, boolean restartConnector) { JsonObject config = new JsonObject(); config.addProperty("domain", "quick-setup"); config.addProperty("preset", preset); @@ -347,13 +370,13 @@ private static TaskResult quick(String preset, Map options, Stri options.forEach(values::addProperty); config.add("options", values); return new TaskResult(true, "OK", "Operation completed", revision, config, List.copyOf(changes), - reloaded, false); + reloaded, false, restartConnector); } private static TaskResult failure(String code, String message) { return failure(code, message, false); } private static TaskResult failure(String code, String message, boolean rolledBack) { return new TaskResult(false, code, message == null ? "Operation failed" : message, null, null, - List.of(), false, rolledBack); + List.of(), false, rolledBack, false); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 7fc837beb0..be8227e945 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -66,5 +66,25 @@ class BackendConfigurationServiceTest { "voteDelay", "24h")); assertTrue(site.proposal().content().contains("PMC:")); assertTrue(site.proposal().content().contains("PlanetMinecraft.com")); + + BackendConfigurationService.QuickPreview reward = service.previewQuickSetup("easy-reward", Map.of( + "scope", "site", "name", "PMC", "command", "eco give %player% 100", + "message", "&aThanks for voting!")); + assertTrue(reward.proposal().content().contains("eco give %player% 100")); + + Files.writeString(directory.resolve("Config.yml"), "ProcessRewards: false\nAutoCreateVoteSites: false\n" + + "ExtraAllSitesCheck: false\nCountFakeVotes: false\nDisableNoServiceSiteMessage: false\n" + + "DisableUpdateChecking: false\n"); + BackendConfigurationService.QuickPreview common = service.previewQuickSetup("common-settings", Map.of( + "processRewards", "true", "autoCreateVoteSites", "true", "extraAllSitesCheck", "true", + "countFakeVotes", "true", "disableNoServiceSiteMessage", "false", + "disableUpdateChecking", "false")); + assertTrue(common.proposal().content().contains("ExtraAllSitesCheck: true")); + + Files.writeString(directory.resolve("SpecialRewards.yml"), "VoteParty:\n Enabled: false\n"); + BackendConfigurationService.QuickPreview party = service.previewQuickSetup("vote-party", Map.of( + "votesRequired", "25", "command", "give %player% diamond 1", "broadcast", "Party!", + "giveAllPlayers", "false", "onlineOnly", "true")); + assertTrue(party.proposal().content().contains("VotesRequired: 25")); } } diff --git a/docs/control-connector.md b/docs/control-connector.md index 142146e07f..395f6e4e3c 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -133,10 +133,12 @@ Control: RequestTimeoutMillis: 10000 ``` -The Bukkit connector owns one daemon worker and performs no Control I/O on the server thread. It negotiates +The Bukkit connector owns one daemon worker and performs no Control I/O on the server thread. It reports a bounded list of +installed plugin names for WebUI command suggestions and negotiates `config.files.v1` and `config.quick-setup.v1`, then polls the same outbound operation queue as proxies. File apply schedules the VotingPlugin reload on the Bukkit thread and waits only on the connector worker. Control failure never blocks votes, -joins, commands, or plugin shutdown. +joins, commands, or plugin shutdown. A successful `Config.yml` apply reports its result first, then recreates the connector +so changes to `Control.Backend` take effect without a full server restart. ## Discovery semantics @@ -165,8 +167,11 @@ if reload fails. Returned YAML is normalized and masks password/secret/token/API with `__VOTINGPLUGIN_CONTROL_REDACTED__`; leaving the marker unchanged preserves the local value. A replacement secret may be submitted through the authenticated preview, but is never returned or audited. -Quick setups currently cover standalone backend mode, proxy-connected backend mode with an explicit server identity, and -adding/updating a vote site. They use the same preview, revision, approval, backup, reload, and rollback path—not a bypass. +Quick setups cover standalone backend mode, proxy-connected backend mode with an explicit server identity, adding/updating +a vote site, an easy per-site or every-site command/message reward, six common operational toggles, and vote-party basics. +Detected Essentials/EssentialsX, CMI, and LuckPerms installations add editable command suggestions alongside generic +Minecraft rewards. Plugin detection does not inspect third-party configuration or versions. Every shortcut uses the same +preview, revision, approval, backup, reload, and rollback path—not a bypass. An admin must select capable online nodes, preview the change, and confirm the single-use approval generated for that exact successful preview. Nodes claim work over their existing outbound connection, From bd9c98d5252e26d57c8318a773725136b3123e21 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 17:07:01 -0600 Subject: [PATCH 11/73] Fix configuration apply lifecycle races --- .../votingplugin/VotingPluginMain.java | 7 ++++ .../control/BackendConfigurationService.java | 32 +++++++++++-------- .../control/BackendControlConnector.java | 32 +++++++++++-------- .../proxy/VotingPluginProxyConfig.java | 11 +++++++ .../proxy/bungee/BungeeConfig.java | 15 +++++++-- .../ProxyRoutingConfigurationService.java | 12 ++++--- .../proxy/velocity/VelocityConfig.java | 13 ++++++-- .../proxy/control/ControlConnectorTest.java | 2 +- .../ProxyRoutingConfigurationServiceTest.java | 19 ++++++++++- 9 files changed, 105 insertions(+), 38 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index b513a23aee..2d503242dc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1601,6 +1601,13 @@ public synchronized void restartBackendControlConnector() { startBackendControlConnector(); } + /** Recreates proxy transports after Control applies BungeeSettings.yml. */ + public synchronized void restartBackendProxyHandler() { + if (backendProxyHandler != null) backendProxyHandler.close(); + backendProxyHandler = null; + if (bungeeSettings.isUseBungeecoord()) loadBungeeHandler(); + } + private void migrateVoteBroadcast(Config configFile) { ConfigurationSection cfg = configFile.getData(); // If new section exists, do nothing diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 93e2d58643..585b264f8c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -99,21 +99,33 @@ private Preview preview(String fileName, String proposedContent, String current) } public QuickPreview previewQuickSetup(String preset, Map options) throws IOException { - QuickProposal proposal = quickProposal(preset, options); - String current = readRaw(resolve(proposal.fileName()), false); + String fileName = quickSetupFile(preset); + String current = readRaw(resolve(fileName), false); + QuickProposal proposal = quickProposal(preset, options, fileName, current); return new QuickPreview(proposal, revision(current), changes(parse(current), parse(proposal.content()))); } public ApplyResult applyQuickSetup(String preset, Map options, String expectedRevision) throws IOException { - QuickProposal proposal = quickProposal(preset, options); + String fileName = quickSetupFile(preset); + String current = readRaw(resolve(fileName), false); + if (expectedRevision == null || !revision(current).equals(expectedRevision)) throw new StaleRevisionException(); + QuickProposal proposal = quickProposal(preset, options, fileName, current); return apply(proposal.fileName(), proposal.content(), expectedRevision); } - private QuickProposal quickProposal(String preset, Map options) throws IOException { + private static String quickSetupFile(String preset) { + if ("standalone".equals(preset) || "proxy-backend".equals(preset)) return "BungeeSettings.yml"; + if ("vote-site".equals(preset) || "easy-reward".equals(preset)) return "VoteSites.yml"; + if ("common-settings".equals(preset)) return "Config.yml"; + if ("vote-party".equals(preset)) return "SpecialRewards.yml"; + throw new IllegalArgumentException("quick setup preset is unsupported"); + } + + private QuickProposal quickProposal(String preset, Map options, String fileName, + String current) { + YamlConfiguration yaml = parse(current); if ("standalone".equals(preset) || "proxy-backend".equals(preset)) { - String fileName = "BungeeSettings.yml"; - YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); boolean proxy = "proxy-backend".equals(preset); yaml.set("UseBungeecord", proxy); if (proxy) { @@ -124,8 +136,6 @@ private QuickProposal quickProposal(String preset, Map options) return new QuickProposal(fileName, yaml.saveToString()); } if ("vote-site".equals(preset)) { - String fileName = "VoteSites.yml"; - YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); String name = option(options, "name", "[A-Za-z0-9_-]{1,64}"); String root = "VoteSites." + name; yaml.set(root + ".Enabled", true); @@ -140,8 +150,6 @@ private QuickProposal quickProposal(String preset, Map options) return new QuickProposal(fileName, yaml.saveToString()); } if ("easy-reward".equals(preset)) { - String fileName = "VoteSites.yml"; - YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); String scope = options == null ? "site" : options.getOrDefault("scope", "site"); String root; if ("every-site".equals(scope)) { @@ -161,8 +169,6 @@ private QuickProposal quickProposal(String preset, Map options) return new QuickProposal(fileName, yaml.saveToString()); } if ("common-settings".equals(preset)) { - String fileName = "Config.yml"; - YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); yaml.set("ProcessRewards", booleanOption(options, "processRewards")); yaml.set("AutoCreateVoteSites", booleanOption(options, "autoCreateVoteSites")); yaml.set("ExtraAllSitesCheck", booleanOption(options, "extraAllSitesCheck")); @@ -172,8 +178,6 @@ private QuickProposal quickProposal(String preset, Map options) return new QuickProposal(fileName, yaml.saveToString()); } if ("vote-party".equals(preset)) { - String fileName = "SpecialRewards.yml"; - YamlConfiguration yaml = parse(readRaw(resolve(fileName), false)); yaml.set("VoteParty.Enabled", true); yaml.set("VoteParty.VotesRequired", boundedInteger(option(options, "votesRequired", "[0-9]{1,6}"), 1, 100000)); yaml.set("VoteParty.GiveAllPlayers", booleanOption(options, "giveAllPlayers")); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index bb8bb6ce6a..e93cf49efc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -188,9 +188,13 @@ private void claimAndExecute() throws Exception { JsonObject resultBody = result.json(sessionId); requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", resultBody), 200); - if (result.restartConnector()) { + TaskResult submitted = result; + if (submitted.restartConnector() || submitted.restartBackendProxy()) { synchronized (completed) { completed.remove(operationId); } - plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); + plugin.getServer().getScheduler().runTask(plugin, () -> { + if (submitted.restartBackendProxy()) plugin.restartBackendProxyHandler(); + if (submitted.restartConnector()) plugin.restartBackendControlConnector(); + }); } } @@ -220,20 +224,20 @@ private TaskResult executeFile(String type, JsonObject configuration, JsonObject String fileName = string(configuration, "fileName"); if ("READ".equals(type)) { BackendConfigurationService.Document document = configurations.read(fileName); - return TaskResult.file(document, List.of(), false, false, false); + return TaskResult.file(document, List.of(), false, false, false, false); } String content = string(configuration, "content"); if ("PREVIEW".equals(type)) { BackendConfigurationService.Preview preview = configurations.preview(fileName, content); BackendConfigurationService.Document current = configurations.read(fileName); if (!preview.revision().equals(current.revision())) throw new BackendConfigurationService.StaleRevisionException(); - return TaskResult.file(current, preview.changes(), false, false, false); + return TaskResult.file(current, preview.changes(), false, false, false, false); } if ("APPLY".equals(type)) { BackendConfigurationService.ApplyResult applied = configurations.apply(fileName, content, string(task, "expectedRevision")); return TaskResult.file(applied.document(), applied.changes(), true, applied.rolledBack(), - "Config.yml".equals(fileName)); + "Config.yml".equals(fileName), "BungeeSettings.yml".equals(fileName)); } return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); } @@ -250,7 +254,8 @@ private TaskResult executeQuick(String type, JsonObject configuration, JsonObjec BackendConfigurationService.ApplyResult applied = configurations.applyQuickSetup(preset, options, string(task, "expectedRevision")); return TaskResult.quick(preset, options, applied.document().revision(), applied.changes(), true, - "Config.yml".equals(applied.document().fileName())); + "Config.yml".equals(applied.document().fileName()), + "BungeeSettings.yml".equals(applied.document().fileName())); } return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); } @@ -330,7 +335,7 @@ private record Response(int status, String body) { } private record TaskResult(boolean success, String code, String message, String revision, JsonObject configuration, List changes, boolean reloaded, boolean rolledBack, - boolean restartConnector) { + boolean restartConnector, boolean restartBackendProxy) { private JsonObject json(UUID sessionId) { JsonObject body = new JsonObject(); body.addProperty("sessionId", sessionId.toString()); @@ -348,21 +353,22 @@ private JsonObject json(UUID sessionId) { } private static TaskResult file(BackendConfigurationService.Document document, List changes, - boolean reloaded, boolean rolledBack, boolean restartConnector) { + boolean reloaded, boolean rolledBack, boolean restartConnector, boolean restartBackendProxy) { JsonObject config = new JsonObject(); config.addProperty("domain", "file"); config.addProperty("fileName", document.fileName()); config.addProperty("content", document.content()); return new TaskResult(true, "OK", "Operation completed", document.revision(), config, - List.copyOf(changes), reloaded, rolledBack, restartConnector); + List.copyOf(changes), reloaded, rolledBack, restartConnector, restartBackendProxy); } private static TaskResult quick(String preset, Map options, String revision, List changes, boolean reloaded) { - return quick(preset, options, revision, changes, reloaded, false); + return quick(preset, options, revision, changes, reloaded, false, false); } private static TaskResult quick(String preset, Map options, String revision, - List changes, boolean reloaded, boolean restartConnector) { + List changes, boolean reloaded, boolean restartConnector, + boolean restartBackendProxy) { JsonObject config = new JsonObject(); config.addProperty("domain", "quick-setup"); config.addProperty("preset", preset); @@ -370,13 +376,13 @@ private static TaskResult quick(String preset, Map options, Stri options.forEach(values::addProperty); config.add("options", values); return new TaskResult(true, "OK", "Operation completed", revision, config, List.copyOf(changes), - reloaded, false, restartConnector); + reloaded, false, restartConnector, restartBackendProxy); } private static TaskResult failure(String code, String message) { return failure(code, message, false); } private static TaskResult failure(String code, String message, boolean rolledBack) { return new TaskResult(false, code, message == null ? "Operation failed" : message, null, null, - List.of(), false, rolledBack, false); + List.of(), false, rolledBack, false, false); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java index ed1442da6c..4327673078 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java @@ -16,10 +16,21 @@ default void persistControlProxyRouting(boolean sendVotesToAllServers, List blockedServers, + String expectedRevision) throws IOException { + persistControlProxyRouting(sendVotesToAllServers, blockedServers); + } + /** Restores the most recent Control-created backup. */ default void rollbackControlProxyRouting() throws IOException { throw new UnsupportedOperationException("Control configuration rollback is unavailable"); } + + @SuppressWarnings("serial") + final class StaleControlRevisionException extends IOException { + public StaleControlRevisionException() { super("Control proxy routing revision is stale"); } + } /** Whether the optional local VotingPlugin Control connector is enabled. */ default boolean getControlEnabled() { return false; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index 9cc07ce0fb..a2e532efe0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -12,7 +12,8 @@ import java.util.List; import java.util.Map; -import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.bencodez.votingplugin.proxy.control.ProxyRoutingConfiguration; import lombok.Getter; import net.md_5.bungee.config.Configuration; @@ -513,16 +514,19 @@ public void save() { } @Override - public synchronized void persistControlProxyRouting(boolean sendVotesToAllServers, List blockedServers) - throws IOException { + public synchronized void persistControlProxyRouting(boolean sendVotesToAllServers, List blockedServers, + String expectedRevision) throws IOException { Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-stage"); Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); Configuration candidate = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); + if (!routing(candidate).revision().equals(expectedRevision)) throw new StaleControlRevisionException(); candidate.set("SendVotesToAllServers", sendVotesToAllServers); candidate.set("BlockedServers", List.copyOf(blockedServers)); try { ConfigurationProvider.getProvider(YamlConfiguration.class).save(candidate, stage.toFile()); + Configuration latest = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); + if (!routing(latest).revision().equals(expectedRevision)) throw new StaleControlRevisionException(); Files.copy(target, backup, StandardCopyOption.REPLACE_EXISTING); atomicReplace(stage, target); data = candidate; @@ -531,6 +535,11 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer } } + private static ProxyRoutingConfiguration routing(Configuration configuration) { + return new ProxyRoutingConfiguration(configuration.getBoolean("SendVotesToAllServers", false), + configuration.getStringList("BlockedServers")); + } + @Override public synchronized void rollbackControlProxyRouting() throws IOException { Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java index 264995ac93..e9345ca475 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java @@ -17,9 +17,9 @@ public ProxyRoutingConfigurationService(VotingPluginProxy proxy) { proxy.getConfig().getBlockedServers()); } @Override public Set configuredServers() { return proxy.getAllConfiguredServers(); } - @Override public void persist(ProxyRoutingConfiguration proposal) throws IOException { + @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) throws IOException { proxy.getConfig().persistControlProxyRouting(proposal.sendVotesToAllServers(), - proposal.blockedServers()); + proposal.blockedServers(), expectedRevision); } @Override public void rollback() throws IOException { proxy.getConfig().rollbackControlProxyRouting(); } @Override public void reload() throws Exception { proxy.reloadControlConfiguration(); } @@ -48,7 +48,11 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t if (expectedRevision == null || !read().revision().equals(expectedRevision)) { throw new StaleRevisionException(); } - platform.persist(proposal); + try { + platform.persist(proposal, expectedRevision); + } catch (com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException e) { + throw new StaleRevisionException(); + } try { platform.reload(); } catch (Exception e) { @@ -67,7 +71,7 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t interface Platform { ProxyRoutingConfiguration read(); Set configuredServers(); - void persist(ProxyRoutingConfiguration proposal) throws IOException; + void persist(ProxyRoutingConfiguration proposal, String expectedRevision) throws IOException; void rollback() throws IOException; void reload() throws Exception; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 904c9fad61..eb00871774 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -16,6 +16,7 @@ import com.bencodez.simpleapi.file.velocity.VelocityYMLFile; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.bencodez.votingplugin.proxy.control.ProxyRoutingConfiguration; import org.spongepowered.configurate.ConfigurationNode; import org.spongepowered.configurate.yaml.YamlConfigurationLoader; @@ -36,17 +37,20 @@ public VelocityConfig(File file) { } @Override - public synchronized void persistControlProxyRouting(boolean sendVotesToAllServers, List blockedServers) - throws IOException { + public synchronized void persistControlProxyRouting(boolean sendVotesToAllServers, List blockedServers, + String expectedRevision) throws IOException { Path target = configurationFile.toPath(); Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-stage"); Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); YamlConfigurationLoader sourceLoader = YamlConfigurationLoader.builder().path(target).build(); ConfigurationNode candidate = sourceLoader.load(); + if (!routing(candidate).revision().equals(expectedRevision)) throw new StaleControlRevisionException(); candidate.node("SendVotesToAllServers").set(sendVotesToAllServers); candidate.node("BlockedServers").setList(String.class, List.copyOf(blockedServers)); try { YamlConfigurationLoader.builder().path(stage).build().save(candidate); + ConfigurationNode latest = sourceLoader.load(); + if (!routing(latest).revision().equals(expectedRevision)) throw new StaleControlRevisionException(); Files.copy(target, backup, StandardCopyOption.REPLACE_EXISTING); atomicReplace(stage, target); } finally { @@ -54,6 +58,11 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer } } + private static ProxyRoutingConfiguration routing(ConfigurationNode configuration) throws IOException { + return new ProxyRoutingConfiguration(configuration.node("SendVotesToAllServers").getBoolean(false), + configuration.node("BlockedServers").getList(String.class, List.of())); + } + @Override public synchronized void rollbackControlProxyRouting() throws IOException { Path target = configurationFile.toPath(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 95a6e57cad..12b25af880 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -219,7 +219,7 @@ public CompletableFuture send(Request request) { private static final class NoOpPlatform implements ProxyRoutingConfigurationService.Platform { @Override public ProxyRoutingConfiguration read() { return new ProxyRoutingConfiguration(false, List.of()); } @Override public java.util.Set configuredServers() { return java.util.Set.of(); } - @Override public void persist(ProxyRoutingConfiguration proposal) { } + @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) { } @Override public void rollback() { } @Override public void reload() { } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java index 30ebfdb40f..6cd2d87477 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java @@ -39,16 +39,33 @@ void strictReloadFailureRestoresBackupAndReloadsAgain() throws Exception { assertTrue(platform.reloadCalls == 2); } + @Test + void persistenceRevisionRaceRemainsAStaleRevision() { + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + FakePlatform platform = new FakePlatform(); + platform.current = current; + platform.staleDuringPersist = true; + + assertThrows(ProxyRoutingConfigurationService.StaleRevisionException.class, + () -> new ProxyRoutingConfigurationService(platform).apply( + new ProxyRoutingConfiguration(true, List.of()), current.revision())); + assertTrue(platform.reloadCalls == 0); + } + private static final class FakePlatform implements ProxyRoutingConfigurationService.Platform { private ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); private Set configuredServers = Set.of(); private boolean failFirstReload; private boolean rolledBack; + private boolean staleDuringPersist; private int reloadCalls; @Override public ProxyRoutingConfiguration read() { return current; } @Override public Set configuredServers() { return configuredServers; } - @Override public void persist(ProxyRoutingConfiguration proposal) { current = proposal; } + @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) throws IOException { + if (staleDuringPersist) throw new com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException(); + current = proposal; + } @Override public void rollback() { rolledBack = true; } @Override public void reload() throws Exception { reloadCalls++; From dd75fa9afa0c815dcaea1a5b617905beb0f69255 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 17:39:08 -0600 Subject: [PATCH 12/73] Validate backend control compatibility and transport --- .../votingplugin/VotingPluginMain.java | 19 ++++++-- .../backendproxy/BackendProxyHandler.java | 19 ++++++++ .../control/BackendConfigurationService.java | 11 +++-- .../control/BackendControlConnector.java | 46 +++++++++++-------- .../BackendConfigurationServiceTest.java | 20 ++++++++ .../BackendControlConnectorProtocolTest.java | 13 ++++++ 6 files changed, 103 insertions(+), 25 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 2d503242dc..e50486f307 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1603,9 +1603,22 @@ public synchronized void restartBackendControlConnector() { /** Recreates proxy transports after Control applies BungeeSettings.yml. */ public synchronized void restartBackendProxyHandler() { - if (backendProxyHandler != null) backendProxyHandler.close(); - backendProxyHandler = null; - if (bungeeSettings.isUseBungeecoord()) loadBungeeHandler(); + BackendProxyHandler previous = backendProxyHandler; + if (!bungeeSettings.isUseBungeecoord()) { + backendProxyHandler = null; + if (previous != null) previous.close(); + return; + } + BackendProxyHandler replacement = new BackendProxyHandler(this); + try { + replacement.load(); + replacement.validateTransport(); + } catch (RuntimeException failure) { + replacement.close(); + throw failure; + } + backendProxyHandler = replacement; + if (previous != null) previous.close(); } private void migrateVoteBroadcast(Config configFile) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 00c4d1435a..72098cd162 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -568,6 +568,25 @@ public void onReceiveEnvelope(JsonEnvelope envelope) { presenceManager.start(); } + /** Fails an apply when the selected proxy transport could not be initialized. */ + public void validateTransport() { + if (method == null || globalMessageHandler == null || presenceManager == null) { + throw new IllegalStateException("Backend proxy handler initialization failed"); + } + if (method.equals(BungeeMethod.MYSQL) && backendMysqlMessenger == null) { + throw new IllegalStateException("MySQL backend proxy transport initialization failed"); + } + if (method.equals(BungeeMethod.REDIS) && redisHandler == null) { + throw new IllegalStateException("Redis backend proxy transport initialization failed"); + } + if (method.equals(BungeeMethod.SOCKETS) && (clientHandler == null || socketHandler == null)) { + throw new IllegalStateException("Socket backend proxy transport initialization failed"); + } + if (method.equals(BungeeMethod.MQTT) && mqttHandler == null) { + throw new IllegalStateException("MQTT backend proxy transport initialization failed"); + } + } + /** * Announces a player login to the configured proxy transport. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 585b264f8c..513671d5e0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -32,9 +32,13 @@ public final class BackendConfigurationService { "GUI.yml", "Shop.yml", "BungeeSettings.yml"); private final Path dataDirectory; - private final ReloadAction reload; + private final ApplyAction reload; public BackendConfigurationService(Path dataDirectory, ReloadAction reload) { + this(dataDirectory, ignored -> reload.run()); + } + + public BackendConfigurationService(Path dataDirectory, ApplyAction reload) { this.dataDirectory = dataDirectory.toAbsolutePath().normalize(); this.reload = reload; } @@ -68,7 +72,7 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); move(staging, target); installed = true; - reload.run(); + reload.run(fileName); String applied = readRaw(target, false); return new ApplyResult(new Document(fileName, mask(parse(applied)), revision(applied)), preview.changes(), false); @@ -79,7 +83,7 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte if (installed) { try { Files.copy(backup, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); - reload.run(); + reload.run(fileName); rolledBack = true; } catch (Exception rollbackFailure) { failure.addSuppressed(rollbackFailure); @@ -332,6 +336,7 @@ public record QuickProposal(String fileName, String content) { } public record QuickPreview(QuickProposal proposal, String revision, List changes) { } @FunctionalInterface public interface ReloadAction { void run() throws Exception; } + @FunctionalInterface public interface ApplyAction { void run(String fileName) throws Exception; } @SuppressWarnings("serial") public static final class StaleRevisionException extends RuntimeException { } @SuppressWarnings("serial") public static final class ApplyFailureException extends IOException { private final boolean rolledBack; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index e93cf49efc..9c76cc7d7d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -67,9 +67,10 @@ private BackendControlConnector(VotingPluginMain plugin, Settings settings, Stri executor = Executors.newSingleThreadScheduledExecutor(factory); http = HttpClient.newBuilder().connectTimeout(Duration.ofMillis(settings.connectTimeoutMillis())) .followRedirects(HttpClient.Redirect.NEVER).build(); - configurations = new BackendConfigurationService(plugin.getDataFolder().toPath(), () -> + configurations = new BackendConfigurationService(plugin.getDataFolder().toPath(), fileName -> plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { plugin.reload(); + if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(); return null; }).get(30, TimeUnit.SECONDS)); } @@ -119,11 +120,19 @@ private void cycle() { throw new ConnectorException("protocol mismatch"); } } + operationsAccepted = false; + quickSetupsAccepted = false; if (node != null && node.has("acceptedCapabilities")) { JsonArray accepted = node.getAsJsonArray("acceptedCapabilities"); operationsAccepted = contains(accepted, "config.files.v1"); quickSetupsAccepted = contains(accepted, "config.quick-setup.v1"); } + try { + requireFileCapability(operationsAccepted); + } catch (ConnectorException incompatible) { + registered = false; + throw incompatible; + } registered = true; if (operationsAccepted) claimAndExecute(); if (failures > 0) plugin.getLogger().info("[Control] Bukkit configuration connector recovered"); @@ -189,12 +198,9 @@ private void claimAndExecute() throws Exception { requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", resultBody), 200); TaskResult submitted = result; - if (submitted.restartConnector() || submitted.restartBackendProxy()) { + if (submitted.restartConnector()) { synchronized (completed) { completed.remove(operationId); } - plugin.getServer().getScheduler().runTask(plugin, () -> { - if (submitted.restartBackendProxy()) plugin.restartBackendProxyHandler(); - if (submitted.restartConnector()) plugin.restartBackendControlConnector(); - }); + plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); } } @@ -224,20 +230,20 @@ private TaskResult executeFile(String type, JsonObject configuration, JsonObject String fileName = string(configuration, "fileName"); if ("READ".equals(type)) { BackendConfigurationService.Document document = configurations.read(fileName); - return TaskResult.file(document, List.of(), false, false, false, false); + return TaskResult.file(document, List.of(), false, false, false); } String content = string(configuration, "content"); if ("PREVIEW".equals(type)) { BackendConfigurationService.Preview preview = configurations.preview(fileName, content); BackendConfigurationService.Document current = configurations.read(fileName); if (!preview.revision().equals(current.revision())) throw new BackendConfigurationService.StaleRevisionException(); - return TaskResult.file(current, preview.changes(), false, false, false, false); + return TaskResult.file(current, preview.changes(), false, false, false); } if ("APPLY".equals(type)) { BackendConfigurationService.ApplyResult applied = configurations.apply(fileName, content, string(task, "expectedRevision")); return TaskResult.file(applied.document(), applied.changes(), true, applied.rolledBack(), - "Config.yml".equals(fileName), "BungeeSettings.yml".equals(fileName)); + "Config.yml".equals(fileName)); } return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); } @@ -254,8 +260,7 @@ private TaskResult executeQuick(String type, JsonObject configuration, JsonObjec BackendConfigurationService.ApplyResult applied = configurations.applyQuickSetup(preset, options, string(task, "expectedRevision")); return TaskResult.quick(preset, options, applied.document().revision(), applied.changes(), true, - "Config.yml".equals(applied.document().fileName()), - "BungeeSettings.yml".equals(applied.document().fileName())); + "Config.yml".equals(applied.document().fileName())); } return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); } @@ -304,6 +309,10 @@ private static boolean contains(JsonArray accepted, String expected) { return false; } + static void requireFileCapability(boolean accepted) { + if (!accepted) throw new ConnectorException("required config.files.v1 capability was not accepted"); + } + private static String string(JsonObject object, String name) { if (object == null || !object.has(name) || object.get(name).isJsonNull() || !object.get(name).isJsonPrimitive()) throw new IllegalArgumentException(name + " is required"); @@ -335,7 +344,7 @@ private record Response(int status, String body) { } private record TaskResult(boolean success, String code, String message, String revision, JsonObject configuration, List changes, boolean reloaded, boolean rolledBack, - boolean restartConnector, boolean restartBackendProxy) { + boolean restartConnector) { private JsonObject json(UUID sessionId) { JsonObject body = new JsonObject(); body.addProperty("sessionId", sessionId.toString()); @@ -353,22 +362,21 @@ private JsonObject json(UUID sessionId) { } private static TaskResult file(BackendConfigurationService.Document document, List changes, - boolean reloaded, boolean rolledBack, boolean restartConnector, boolean restartBackendProxy) { + boolean reloaded, boolean rolledBack, boolean restartConnector) { JsonObject config = new JsonObject(); config.addProperty("domain", "file"); config.addProperty("fileName", document.fileName()); config.addProperty("content", document.content()); return new TaskResult(true, "OK", "Operation completed", document.revision(), config, - List.copyOf(changes), reloaded, rolledBack, restartConnector, restartBackendProxy); + List.copyOf(changes), reloaded, rolledBack, restartConnector); } private static TaskResult quick(String preset, Map options, String revision, List changes, boolean reloaded) { - return quick(preset, options, revision, changes, reloaded, false, false); + return quick(preset, options, revision, changes, reloaded, false); } private static TaskResult quick(String preset, Map options, String revision, - List changes, boolean reloaded, boolean restartConnector, - boolean restartBackendProxy) { + List changes, boolean reloaded, boolean restartConnector) { JsonObject config = new JsonObject(); config.addProperty("domain", "quick-setup"); config.addProperty("preset", preset); @@ -376,13 +384,13 @@ private static TaskResult quick(String preset, Map options, Stri options.forEach(values::addProperty); config.add("options", values); return new TaskResult(true, "OK", "Operation completed", revision, config, List.copyOf(changes), - reloaded, false, restartConnector, restartBackendProxy); + reloaded, false, restartConnector); } private static TaskResult failure(String code, String message) { return failure(code, message, false); } private static TaskResult failure(String code, String message, boolean rolledBack) { return new TaskResult(false, code, message == null ? "Operation failed" : message, null, null, - List.of(), false, rolledBack, false, false); + List.of(), false, rolledBack, false); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index be8227e945..678ecb93b7 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -51,6 +51,26 @@ class BackendConfigurationServiceTest { assertThrows(IllegalArgumentException.class, () -> service.preview("Config.yml", "bad: [yaml")); } + @Test void transportValidationFailureRollsBackBeforeApplyReturns() throws Exception { + Path settings = directory.resolve("BungeeSettings.yml"); + Files.writeString(settings, "UseBungeecord: false\nBungeeMethod: PLUGINMESSAGING\n"); + AtomicInteger reloads = new AtomicInteger(); + BackendConfigurationService service = new BackendConfigurationService(directory, + (BackendConfigurationService.ApplyAction) fileName -> { + assertEquals("BungeeSettings.yml", fileName); + if (reloads.incrementAndGet() == 1) throw new IllegalStateException("transport unavailable"); + }); + BackendConfigurationService.Document before = service.read("BungeeSettings.yml"); + BackendConfigurationService.ApplyFailureException failure = assertThrows( + BackendConfigurationService.ApplyFailureException.class, + () -> service.apply("BungeeSettings.yml", + "UseBungeecord: true\nBungeeMethod: MQTT\n", before.revision())); + + assertTrue(failure.rolledBack()); + assertEquals(2, reloads.get()); + assertTrue(service.read("BungeeSettings.yml").content().contains("PLUGINMESSAGING")); + } + @Test void quickSetupsProduceReviewableBackendAndVoteSiteChanges() throws Exception { Files.writeString(directory.resolve("BungeeSettings.yml"), "UseBungeecord: false\nServer: PleaseSet\n"); Files.writeString(directory.resolve("VoteSites.yml"), "VoteSites: {}\n"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java new file mode 100644 index 0000000000..bc90a375fc --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -0,0 +1,13 @@ +package com.bencodez.votingplugin.control; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import org.junit.jupiter.api.Test; + +class BackendControlConnectorProtocolTest { + @Test void registrationRequiresFileControlButAllowsQuickSetupToRemainOptional() { + assertThrows(RuntimeException.class, () -> BackendControlConnector.requireFileCapability(false)); + assertDoesNotThrow(() -> BackendControlConnector.requireFileCapability(true)); + } +} From 1efc0fc30a927742c66156cba546104f8b8376be Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 18:05:13 -0600 Subject: [PATCH 13/73] Harden backend capability and path handling --- .../control/BackendConfigurationService.java | 20 +++++++++++++++---- .../control/BackendControlConnector.java | 17 +++++++++------- .../BackendConfigurationServiceTest.java | 15 ++++++++++++++ .../BackendControlConnectorProtocolTest.java | 17 ++++++++++++++++ 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 513671d5e0..5573deb9df 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -195,19 +195,31 @@ private QuickProposal quickProposal(String preset, Map options, throw new IllegalArgumentException("quick setup preset is unsupported"); } - private Path resolve(String fileName) { + private Path resolve(String fileName) throws IOException { if (fileName == null || (!TOP_LEVEL.contains(fileName) && !fileName.matches("VoteSites/[A-Za-z0-9._-]{1,100}\\.yml"))) { throw new IllegalArgumentException("configuration file is not managed"); } - Path target = dataDirectory.resolve(fileName).normalize(); - if (!target.startsWith(dataDirectory)) throw new IllegalArgumentException("configuration path escapes data folder"); + Path requested = dataDirectory.resolve(fileName).normalize(); + if (!requested.startsWith(dataDirectory)) { + throw new IllegalArgumentException("configuration path escapes data folder"); + } + Path root = dataDirectory.toRealPath(); + Path parent = requested.getParent(); + if (parent == null) throw new IOException("configuration parent is unavailable"); + Path realParent = parent.toRealPath(); + if (!realParent.startsWith(root)) throw new IOException("configuration path escapes data folder"); + Path target = realParent.resolve(requested.getFileName()).normalize(); + if (!target.startsWith(root) || Files.isSymbolicLink(target)) { + throw new IOException("symbolic configuration paths are not allowed"); + } return target; } private static String readRaw(Path path, boolean allowMissing) throws IOException { if (!Files.exists(path) && allowMissing) return ""; - if (!Files.isRegularFile(path) || Files.size(path) > MAX_CONTENT_BYTES) { + if (!Files.isRegularFile(path, java.nio.file.LinkOption.NOFOLLOW_LINKS) + || Files.size(path) > MAX_CONTENT_BYTES) { throw new IOException("configuration file is missing or too large"); } byte[] bytes = Files.readAllBytes(path); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 9c76cc7d7d..f37ae630cf 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -120,13 +120,8 @@ private void cycle() { throw new ConnectorException("protocol mismatch"); } } - operationsAccepted = false; - quickSetupsAccepted = false; - if (node != null && node.has("acceptedCapabilities")) { - JsonArray accepted = node.getAsJsonArray("acceptedCapabilities"); - operationsAccepted = contains(accepted, "config.files.v1"); - quickSetupsAccepted = contains(accepted, "config.quick-setup.v1"); - } + operationsAccepted = negotiatedCapability(node, "config.files.v1", operationsAccepted); + quickSetupsAccepted = negotiatedCapability(node, "config.quick-setup.v1", quickSetupsAccepted); try { requireFileCapability(operationsAccepted); } catch (ConnectorException incompatible) { @@ -151,6 +146,9 @@ private void cycle() { } private JsonObject register() throws Exception { + // A registration must explicitly establish required capabilities. Heartbeats may omit the unchanged set. + operationsAccepted = false; + quickSetupsAccepted = false; JsonObject body = sessionBody(); body.addProperty("nodeId", settings.nodeId()); body.addProperty("displayName", settings.nodeId()); @@ -309,6 +307,11 @@ private static boolean contains(JsonArray accepted, String expected) { return false; } + static boolean negotiatedCapability(JsonObject node, String capability, boolean current) { + if (node == null || !node.has("acceptedCapabilities")) return current; + return contains(node.getAsJsonArray("acceptedCapabilities"), capability); + } + static void requireFileCapability(boolean accepted) { if (!accepted) throw new ConnectorException("required config.files.v1 capability was not accepted"); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 678ecb93b7..10927de89b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -51,6 +51,21 @@ class BackendConfigurationServiceTest { assertThrows(IllegalArgumentException.class, () -> service.preview("Config.yml", "bad: [yaml")); } + @Test void rejectsManagedFileAndDirectorySymlinkEscapes() throws Exception { + Path root = Files.createDirectory(directory.resolve("plugin-data")); + Path outside = Files.createDirectory(directory.resolve("outside")); + Path externalConfig = outside.resolve("Config.yml"); + Files.writeString(externalConfig, "Feature: external\n"); + Files.createSymbolicLink(root.resolve("Config.yml"), externalConfig); + BackendConfigurationService service = new BackendConfigurationService(root, () -> { }); + + assertThrows(java.io.IOException.class, () -> service.read("Config.yml")); + Files.createDirectory(outside.resolve("sites")); + Files.writeString(outside.resolve("sites/External.yml"), "VoteSites: {}\n"); + Files.createSymbolicLink(root.resolve("VoteSites"), outside.resolve("sites")); + assertThrows(java.io.IOException.class, () -> service.read("VoteSites/External.yml")); + } + @Test void transportValidationFailureRollsBackBeforeApplyReturns() throws Exception { Path settings = directory.resolve("BungeeSettings.yml"); Files.writeString(settings, "UseBungeecord: false\nBungeeMethod: PLUGINMESSAGING\n"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index bc90a375fc..46aefe56c6 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -1,13 +1,30 @@ package com.bencodez.votingplugin.control; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import org.junit.jupiter.api.Test; +import com.google.gson.JsonArray; +import com.google.gson.JsonObject; + class BackendControlConnectorProtocolTest { @Test void registrationRequiresFileControlButAllowsQuickSetupToRemainOptional() { assertThrows(RuntimeException.class, () -> BackendControlConnector.requireFileCapability(false)); assertDoesNotThrow(() -> BackendControlConnector.requireFileCapability(true)); } + + @Test void heartbeatRetainsOmittedCapabilitiesAndHonorsExplicitReplacement() { + JsonObject omitted = new JsonObject(); + assertTrue(BackendControlConnector.negotiatedCapability(omitted, "config.files.v1", true)); + + JsonObject explicit = new JsonObject(); + JsonArray accepted = new JsonArray(); + accepted.add("config.quick-setup.v1"); + explicit.add("acceptedCapabilities", accepted); + assertFalse(BackendControlConnector.negotiatedCapability(explicit, "config.files.v1", true)); + assertTrue(BackendControlConnector.negotiatedCapability(explicit, "config.quick-setup.v1", false)); + } } From de4099f5d73253b2274139830a7e6526574c016f Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 18:19:42 -0600 Subject: [PATCH 14/73] Close replaced proxy transport handlers --- .../votingplugin/VotingPluginMain.java | 24 ++++++++++++ .../backendproxy/BackendProxyHandler.java | 10 ++--- .../votingplugin/proxy/VotingPluginProxy.java | 38 +++++++++++++++---- .../proxy/VotingPluginProxyLifecycleTest.java | 30 +++++++++++++++ 4 files changed, 87 insertions(+), 15 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index e50486f307..b1f9499344 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -20,6 +20,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -56,6 +57,8 @@ import com.bencodez.advancedcore.api.user.UserDataFetchMode; import com.bencodez.advancedcore.api.user.UserStartup; import com.bencodez.simpleapi.file.YMLConfig; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessageHandler; import com.bencodez.simpleapi.skull.SkullCache; import com.bencodez.simpleapi.sql.mysql.config.MysqlConfigSpigot; import com.bencodez.simpleapi.time.ParsedDuration; @@ -146,6 +149,8 @@ public class VotingPluginMain extends AdvancedCorePlugin { @Getter private BackendProxyHandler backendProxyHandler; + private final AtomicReference backendPluginMessageTarget = new AtomicReference<>(); + private PluginMessageHandler backendPluginMessageRelay; @Getter private BungeeSettings bungeeSettings; @@ -1621,6 +1626,25 @@ public synchronized void restartBackendProxyHandler() { if (previous != null) previous.close(); } + /** Keeps one plugin-message listener for the plugin lifetime and atomically swaps its active backend handler. */ + public synchronized void activateBackendPluginMessageHandler(GlobalMessageHandler target) { + backendPluginMessageTarget.set(target); + if (backendPluginMessageRelay == null) { + backendPluginMessageRelay = new PluginMessageHandler() { + @Override + public void onReceive(com.bencodez.simpleapi.servercomm.codec.JsonEnvelope envelope) { + GlobalMessageHandler current = backendPluginMessageTarget.get(); + if (current != null) current.onMessage(envelope); + } + }; + getPluginMessaging().add(backendPluginMessageRelay); + } + } + + public void deactivateBackendPluginMessageHandler(GlobalMessageHandler target) { + backendPluginMessageTarget.compareAndSet(target, null); + } + private void migrateVoteBroadcast(Config configFile) { ConfigurationSection cfg = configFile.getData(); // If new section exists, do nothing diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 72098cd162..c83fef9568 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -33,7 +33,6 @@ import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; import com.bencodez.simpleapi.servercomm.mqtt.MqttServerComm; import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; -import com.bencodez.simpleapi.servercomm.pluginmessage.PluginMessageHandler; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; import com.bencodez.simpleapi.servercomm.redis.RedisListener; import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; @@ -205,6 +204,8 @@ public boolean checkGlobalDataTimeValue(DataValue data) { * Closes and cleans up all handlers and connections. */ public void close() { + plugin.deactivateBackendPluginMessageHandler(globalMessageHandler); + if (presenceManager != null) { presenceManager.stop(); } @@ -508,12 +509,7 @@ public void run() { plugin.getPluginMessaging().setDebug(plugin.getBungeeSettings().isBungeeDebug()); - plugin.getPluginMessaging().add(new PluginMessageHandler() { - @Override - public void onReceive(JsonEnvelope envelope) { - globalMessageHandler.onMessage(envelope); - } - }); + plugin.activateBackendPluginMessageHandler(globalMessageHandler); } else if (method.equals(BungeeMethod.SOCKETS)) { encryptionHandler = new EncryptionHandler(plugin.getName(), new File(plugin.getDataFolder(), "secretkey.key")); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 2e71ed9c11..42683703ce 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2326,6 +2326,9 @@ public void onDisable() { if (socketHandler != null) { socketHandler.closeConnection(); } + HashMap clients = clientHandles; + clientHandles = null; + stopSocketClients(clients); if (redisHandler != null) { redisHandler.close(); @@ -2475,15 +2478,34 @@ private void reloadRuntime(boolean restartControlServices) { private void rebuildSocketClients() { HashMap rebuilt = new HashMap<>(); - List blocked = getConfig().getBlockedServers(); - for (String server : getConfig().getSpigotServers()) { - if (blocked.contains(server)) continue; - Map data = getConfig().getSpigotServerConfiguration(server); - String host = data.containsKey("Host") ? (String) data.get("Host") : ""; - int port = data.containsKey("Port") ? (int) data.get("Port") : 1298; - rebuilt.put(server, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); - } + try { + List blocked = getConfig().getBlockedServers(); + for (String server : getConfig().getSpigotServers()) { + if (blocked.contains(server)) continue; + Map data = getConfig().getSpigotServerConfiguration(server); + String host = data.containsKey("Host") ? (String) data.get("Host") : ""; + int port = data.containsKey("Port") ? (int) data.get("Port") : 1298; + rebuilt.put(server, new ClientHandler(host, port, encryptionHandler, getConfig().getDebug())); + } + } catch (RuntimeException failure) { + stopSocketClients(rebuilt); + throw failure; + } + HashMap previous = clientHandles; clientHandles = rebuilt; + stopSocketClients(previous); + } + + static void stopSocketClients(Map clients) { + if (clients == null) return; + for (ClientHandler client : clients.values()) { + if (client == null) continue; + try { + client.stopConnection(); + } catch (RuntimeException ignored) { + // Best effort: one broken client must not prevent the remaining sockets from closing. + } + } } private void warnUnsupportedDedicatedVotingProxyMode() { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java new file mode 100644 index 0000000000..ee9b2ba0db --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -0,0 +1,30 @@ +package com.bencodez.votingplugin.proxy; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; + +class VotingPluginProxyLifecycleTest { + + @Test + void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { + ClientHandler failing = mock(ClientHandler.class); + ClientHandler healthy = mock(ClientHandler.class); + doThrow(new IllegalStateException("already closed")).when(failing).stopConnection(); + Map clients = new LinkedHashMap<>(); + clients.put("failing", failing); + clients.put("healthy", healthy); + + VotingPluginProxy.stopSocketClients(clients); + + verify(failing).stopConnection(); + verify(healthy).stopConnection(); + } +} From 38bf0a35569d1b0583dfffc2b40e8a2c0a04f01f Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 18:29:55 -0600 Subject: [PATCH 15/73] Close Redis and preserve proxy config edits --- .../backendproxy/BackendProxyHandler.java | 6 ++++ .../proxy/bungee/BungeeConfig.java | 28 +++++++++++++------ .../proxy/velocity/VelocityConfig.java | 26 ++++++++++++----- 3 files changed, 45 insertions(+), 15 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index c83fef9568..fff6448c42 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -213,6 +213,12 @@ public void close() { if (backendMysqlMessenger != null) { backendMysqlMessenger.shutdown(); } + if (redisHandler != null) { + redisHandler.close(); + } + if (redisThread != null) { + redisThread.interrupt(); + } if (socketHandler != null) { socketHandler.closeConnection(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index a2e532efe0..ef834fcfbe 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -518,20 +518,32 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer String expectedRevision) throws IOException { Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-stage"); + Path backupStage = null; Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); - Configuration candidate = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); - if (!routing(candidate).revision().equals(expectedRevision)) throw new StaleControlRevisionException(); - candidate.set("SendVotesToAllServers", sendVotesToAllServers); - candidate.set("BlockedServers", List.copyOf(blockedServers)); try { - ConfigurationProvider.getProvider(YamlConfiguration.class).save(candidate, stage.toFile()); + backupStage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-backup-stage"); + byte[] sourceSnapshot = Files.readAllBytes(target); Configuration latest = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); - if (!routing(latest).revision().equals(expectedRevision)) throw new StaleControlRevisionException(); - Files.copy(target, backup, StandardCopyOption.REPLACE_EXISTING); + if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target)) + || !routing(latest).revision().equals(expectedRevision)) { + throw new StaleControlRevisionException(); + } + latest.set("SendVotesToAllServers", sendVotesToAllServers); + latest.set("BlockedServers", List.copyOf(blockedServers)); + ConfigurationProvider.getProvider(YamlConfiguration.class).save(latest, stage.toFile()); + if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + Files.write(backupStage, sourceSnapshot); + if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + atomicReplace(backupStage, backup); atomicReplace(stage, target); - data = candidate; + data = latest; } finally { Files.deleteIfExists(stage); + if (backupStage != null) Files.deleteIfExists(backupStage); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index eb00871774..f24f322cdf 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -41,20 +41,32 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer String expectedRevision) throws IOException { Path target = configurationFile.toPath(); Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-stage"); + Path backupStage = null; Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); YamlConfigurationLoader sourceLoader = YamlConfigurationLoader.builder().path(target).build(); - ConfigurationNode candidate = sourceLoader.load(); - if (!routing(candidate).revision().equals(expectedRevision)) throw new StaleControlRevisionException(); - candidate.node("SendVotesToAllServers").set(sendVotesToAllServers); - candidate.node("BlockedServers").setList(String.class, List.copyOf(blockedServers)); try { - YamlConfigurationLoader.builder().path(stage).build().save(candidate); + backupStage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-backup-stage"); + byte[] sourceSnapshot = Files.readAllBytes(target); ConfigurationNode latest = sourceLoader.load(); - if (!routing(latest).revision().equals(expectedRevision)) throw new StaleControlRevisionException(); - Files.copy(target, backup, StandardCopyOption.REPLACE_EXISTING); + if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target)) + || !routing(latest).revision().equals(expectedRevision)) { + throw new StaleControlRevisionException(); + } + latest.node("SendVotesToAllServers").set(sendVotesToAllServers); + latest.node("BlockedServers").setList(String.class, List.copyOf(blockedServers)); + YamlConfigurationLoader.builder().path(stage).build().save(latest); + if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + Files.write(backupStage, sourceSnapshot); + if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + atomicReplace(backupStage, backup); atomicReplace(stage, target); } finally { Files.deleteIfExists(stage); + if (backupStage != null) Files.deleteIfExists(backupStage); } } From 082532991877b151ce8c54eb910650b97100dee0 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 18:43:00 -0600 Subject: [PATCH 16/73] Harden hosted and connector lifecycles --- .../backendproxy/BackendProxyHandler.java | 7 +++ .../control/BackendControlConnector.java | 12 ++--- .../votingplugin/proxy/VotingPluginProxy.java | 7 +++ .../proxy/control/ControlConnector.java | 19 +------ .../proxy/control/HostedControlManager.java | 32 +++++++++--- .../util/ControlCredentialFile.java | 52 +++++++++++++++++++ .../control/HostedControlManagerTest.java | 34 ++++++++++++ .../util/ControlCredentialFileTest.java | 30 +++++++++++ 8 files changed, 160 insertions(+), 33 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/util/ControlCredentialFile.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/util/ControlCredentialFileTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index fff6448c42..6e9a9ce701 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -219,6 +219,13 @@ public void close() { if (redisThread != null) { redisThread.interrupt(); } + if (mqttHandler != null) { + try { + mqttHandler.disconnect(); + } catch (Exception e) { + plugin.getLogger().warning("Unable to disconnect the replaced MQTT backend transport"); + } + } if (socketHandler != null) { socketHandler.closeConnection(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index f37ae630cf..a0bd1f36a6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -7,7 +7,6 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; @@ -27,6 +26,7 @@ import org.bukkit.configuration.ConfigurationSection; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.util.ControlCredentialFile; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -88,14 +88,8 @@ public static BackendControlConnector create(VotingPluginMain plugin) throws IOE throw new IllegalArgumentException("Control.Backend.Endpoint must be an HTTP(S) origin"); } Path root = plugin.getDataFolder().toPath().toAbsolutePath().normalize(); - Path credentialPath = root.resolve(control.getString("CredentialFile", "control-credential.txt")).normalize(); - if (!credentialPath.startsWith(root) || !Files.isRegularFile(credentialPath) || Files.size(credentialPath) > 512) { - throw new IOException("Control backend credential file is missing or invalid"); - } - String credential = Files.readString(credentialPath, StandardCharsets.UTF_8).trim(); - if (credential.isEmpty() || credential.length() > 512 || credential.indexOf('\r') >= 0 || credential.indexOf('\n') >= 0) { - throw new IOException("Control backend credential file is missing or invalid"); - } + String credential = ControlCredentialFile.read(root, + control.getString("CredentialFile", "control-credential.txt")); Settings settings = new Settings(nodeId, endpoint, bounded(control.getInt("HeartbeatSeconds", 30), 10, 300, "HeartbeatSeconds"), bounded(control.getInt("ConnectTimeoutMillis", 3000), 500, 30000, "ConnectTimeoutMillis"), diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 42683703ce..88070cb4fd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2337,6 +2337,13 @@ public void onDisable() { redisPublisherPool.close(); redisPublisherPool = null; } + if (mqttHandler != null) { + try { + mqttHandler.disconnect(); + } catch (Exception e) { + logSevere("Unable to disconnect the MQTT proxy transport"); + } + } bungeeTimeChecker.shutdown(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 534ad0906c..e8065496f3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -7,7 +7,6 @@ import java.net.http.HttpRequest; import java.net.http.HttpResponse; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -36,6 +35,7 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus; +import com.bencodez.votingplugin.util.ControlCredentialFile; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -105,11 +105,7 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio if (credentialName == null || credentialName.isBlank()) { throw new IllegalArgumentException("Control.CredentialFile must be set"); } - Path credentialFile = dataDirectory.resolve(credentialName).normalize(); - if (!credentialFile.startsWith(dataDirectory)) { - throw new IllegalArgumentException("Control.CredentialFile must stay within the plugin data folder"); - } - String credential = readCredential(credentialFile); + String credential = ControlCredentialFile.read(dataDirectory, credentialName); String endpointValue = config.getControlEndpoint(); if (endpointValue == null || endpointValue.isBlank()) { throw new IllegalArgumentException("Control.Endpoint must be set"); @@ -474,17 +470,6 @@ public void close() { transport.close(); } - private static String readCredential(Path path) throws IOException { - if (!Files.isRegularFile(path) || Files.size(path) > 512) { - throw new IOException("Control credential file is missing or invalid"); - } - String credential = Files.readString(path, StandardCharsets.UTF_8).trim(); - if (credential.isEmpty() || credential.length() > 512) { - throw new IOException("Control credential file is missing or invalid"); - } - return credential; - } - private static JsonObject parseObject(String body) { try { JsonElement element = JsonParser.parseString(body); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index c068a1b794..37e16cec29 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -55,6 +55,7 @@ public final class HostedControlManager implements AutoCloseable { private final LongSupplier nanoTime; private final Consumer logger; private final AtomicBoolean workInProgress = new AtomicBoolean(); + private final Object processLifecycle = new Object(); private volatile boolean closed; private volatile Status status = Status.STARTING; private volatile ManagedProcess managedProcess; @@ -213,9 +214,12 @@ private void rollback() throws IOException { private ManagedProcess launch(Path artifact) throws IOException { status = Status.STARTING_PROCESS; - ManagedProcess process = launcher.launch(settings, artifact); - managedProcess = process; - return process; + synchronized (processLifecycle) { + if (closed) throw new IOException("Hosted Control manager is closed"); + ManagedProcess process = launcher.launch(settings, artifact); + managedProcess = process; + return process; + } } private boolean awaitHealthy(ManagedProcess process) throws InterruptedException { @@ -294,18 +298,32 @@ public void closeAndWait() { } private void close(boolean waitForProcess) { - closed = true; - status = Status.STOPPED; + ManagedProcess process; + synchronized (processLifecycle) { + closed = true; + status = Status.STOPPED; + process = managedProcess; + managedProcess = null; + } Future task = activeTask; if (task != null) { task.cancel(true); } - ManagedProcess process = managedProcess; - managedProcess = null; if (process != null) { stopProcess(process, waitForProcess); } executor.shutdownNow(); + if (waitForProcess) { + long waitSeconds = (long) settings.downloadTimeoutSeconds() + settings.startupTimeoutSeconds() + 5L; + try { + if (!executor.awaitTermination(waitSeconds, TimeUnit.SECONDS)) { + throw new IllegalStateException("Hosted Control worker did not stop before reload"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for the hosted Control worker", e); + } + } } private static void stopProcess(ManagedProcess process, boolean wait) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/ControlCredentialFile.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/ControlCredentialFile.java new file mode 100644 index 0000000000..9f54c6620f --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/ControlCredentialFile.java @@ -0,0 +1,52 @@ +package com.bencodez.votingplugin.util; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.CodingErrorAction; +import java.nio.charset.StandardCharsets; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +/** Symlink-safe, bounded credential loading shared by proxy and Bukkit Control connectors. */ +public final class ControlCredentialFile { + private static final int MAX_BYTES = 512; + + private ControlCredentialFile() { } + + public static String read(Path rootDirectory, String configuredPath) throws IOException { + if (configuredPath == null || configuredPath.isBlank()) throw invalid(); + Path lexicalRoot = rootDirectory.toAbsolutePath().normalize(); + Path requested = lexicalRoot.resolve(configuredPath).normalize(); + if (!requested.startsWith(lexicalRoot) || requested.getParent() == null) throw invalid(); + Path realRoot = lexicalRoot.toRealPath(); + Path realParent = requested.getParent().toRealPath(); + if (!realParent.startsWith(realRoot)) throw invalid(); + Path target = realParent.resolve(requested.getFileName()); + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target)) throw invalid(); + + ByteBuffer bytes = ByteBuffer.allocate(MAX_BYTES + 1); + try (SeekableByteChannel channel = Files.newByteChannel(target, StandardOpenOption.READ, + LinkOption.NOFOLLOW_LINKS)) { + while (bytes.hasRemaining() && channel.read(bytes) > 0) { } + } + if (bytes.position() == 0 || bytes.position() > MAX_BYTES) throw invalid(); + bytes.flip(); + String credential; + try { + credential = StandardCharsets.UTF_8.newDecoder().onMalformedInput(CodingErrorAction.REPORT) + .onUnmappableCharacter(CodingErrorAction.REPORT).decode(bytes).toString().trim(); + } catch (java.nio.charset.CharacterCodingException e) { + throw invalid(); + } + if (credential.isEmpty() || credential.length() > MAX_BYTES || credential.indexOf('\r') >= 0 + || credential.indexOf('\n') >= 0) throw invalid(); + return credential; + } + + private static IOException invalid() { + return new IOException("Control credential file is missing or invalid"); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 5585f0906b..ac518df3f5 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -17,6 +17,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.CountDownLatch; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; @@ -142,6 +143,39 @@ void digestMismatchNeverActivatesOrLaunchesDownloadedCode() throws Exception { manager.close(); } + @Test + void closeAndWaitDoesNotReturnUntilArtifactWorkerStops() throws Exception { + byte[] release = "release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control/control.jar"); + HostedControlManager.Settings settings = settings(jar, root.resolve("control/data"), true, false, + digest(release), 1); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager manager = new HostedControlManager(settings, executor, (source, target, maximum, timeout) -> { + entered.countDown(); + while (releaseWorker.getCount() > 0) { + try { + releaseWorker.await(); + } catch (InterruptedException ignored) { } + } + Files.write(target, release); + }, new FakeLauncher(), (endpoint, timeout) -> true, millis -> { }, System::nanoTime, message -> { }); + manager.start(); + assertTrue(entered.await(2, TimeUnit.SECONDS)); + CountDownLatch closingStarted = new CountDownLatch(1); + CompletableFuture closing = CompletableFuture.runAsync(() -> { + closingStarted.countDown(); + manager.closeAndWait(); + }); + assertTrue(closingStarted.await(2, TimeUnit.SECONDS)); + assertFalse(closing.isDone()); + releaseWorker.countDown(); + closing.get(2, TimeUnit.SECONDS); + assertTrue(executor.isTerminated()); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/util/ControlCredentialFileTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/util/ControlCredentialFileTest.java new file mode 100644 index 0000000000..df0f475de1 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/util/ControlCredentialFileTest.java @@ -0,0 +1,30 @@ +package com.bencodez.votingplugin.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ControlCredentialFileTest { + @TempDir Path directory; + + @Test void readsContainedFileAndRejectsFileDirectoryAndTraversalEscapes() throws Exception { + Path root = Files.createDirectory(directory.resolve("plugin")); + Path outside = Files.createDirectory(directory.resolve("outside")); + Files.writeString(root.resolve("credential.txt"), "vpctl_node_valid\n"); + assertEquals("vpctl_node_valid", ControlCredentialFile.read(root, "credential.txt")); + + Path external = outside.resolve("external.txt"); + Files.writeString(external, "vpctl_node_external"); + Files.createSymbolicLink(root.resolve("linked.txt"), external); + assertThrows(java.io.IOException.class, () -> ControlCredentialFile.read(root, "linked.txt")); + Files.createSymbolicLink(root.resolve("linked-directory"), outside); + assertThrows(java.io.IOException.class, + () -> ControlCredentialFile.read(root, "linked-directory/external.txt")); + assertThrows(java.io.IOException.class, () -> ControlCredentialFile.read(root, "../outside/external.txt")); + } +} From 8b324debae3c378188a42bf66a4081276ccfd9b6 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 18:48:15 -0600 Subject: [PATCH 17/73] Await forced hosted process shutdown --- .../proxy/control/HostedControlManager.java | 6 +++- .../control/HostedControlManagerTest.java | 30 +++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 37e16cec29..86778119b8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -326,16 +326,20 @@ private void close(boolean waitForProcess) { } } - private static void stopProcess(ManagedProcess process, boolean wait) { + static void stopProcess(ManagedProcess process, boolean wait) { process.destroy(); if (wait) { try { if (!process.waitFor(3, TimeUnit.SECONDS)) { process.destroyForcibly(); + if (!process.waitFor(3, TimeUnit.SECONDS)) { + throw new IllegalStateException("Hosted Control process did not stop after forced termination"); + } } } catch (InterruptedException e) { process.destroyForcibly(); Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while stopping the hosted Control process", e); } } else { process.onExit().orTimeout(3, TimeUnit.SECONDS).exceptionally(failure -> { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index ac518df3f5..a3d4c017c5 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -176,6 +176,16 @@ void closeAndWaitDoesNotReturnUntilArtifactWorkerStops() throws Exception { assertTrue(executor.isTerminated()); } + @Test + void forcedProcessTerminationIsAwaitedBeforeReturning() { + StubbornProcess process = new StubbornProcess(); + + HostedControlManager.stopProcess(process, true); + + assertTrue(process.forciblyDestroyed); + assertEquals(2, process.waits.get()); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); @@ -215,4 +225,24 @@ private static final class FakeProcess implements HostedControlManager.ManagedPr @Override public boolean waitFor(long timeout, TimeUnit unit) { return !alive; } @Override public CompletableFuture onExit() { return exit; } } + + private static final class StubbornProcess implements HostedControlManager.ManagedProcess { + private final AtomicInteger waits = new AtomicInteger(); + private final CompletableFuture exit = new CompletableFuture<>(); + private volatile boolean alive = true; + private volatile boolean forciblyDestroyed; + + @Override public boolean isAlive() { return alive; } + @Override public void destroy() { } + @Override public void destroyForcibly() { + forciblyDestroyed = true; + alive = false; + exit.complete(null); + } + @Override public boolean waitFor(long timeout, TimeUnit unit) { + waits.incrementAndGet(); + return !alive; + } + @Override public CompletableFuture onExit() { return exit; } + } } From a83cae462961c05fa7f078615e6ddc5d5efcfea2 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 19:01:24 -0600 Subject: [PATCH 18/73] Serialize socket transport replacement --- .../backendproxy/BackendProxyHandler.java | 4 ++ .../votingplugin/proxy/VotingPluginProxy.java | 41 +++++++------- .../proxy/multiproxy/MultiProxyHandler.java | 21 ++++++- .../proxy/VotingPluginProxyLifecycleTest.java | 56 +++++++++++++++++++ .../MultiProxyHandlerLifecycleTest.java | 30 ++++++++++ 5 files changed, 130 insertions(+), 22 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 6e9a9ce701..b032a47db7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -515,6 +515,10 @@ public void run() { } else if (method.equals(BungeeMethod.PLUGINMESSAGING)) { plugin.registerBungeeChannels(plugin.getBungeeSettings().getPluginMessagingChannel()); + // PluginMessaging outlives this transport handler. Clear the previous + // configuration before optionally installing a replacement so a live + // encrypted-to-plaintext reload cannot retain stale encryption state. + plugin.getPluginMessaging().setEncryptionHandler(null); if (plugin.getBungeeSettings().isPluginMessageEncryption()) { encryptionHandler = new EncryptionHandler(plugin.getName(), new File(plugin.getDataFolder(), "secretkey.key")); plugin.getPluginMessaging().setEncryptionHandler(encryptionHandler); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 88070cb4fd..576235159e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -548,17 +548,7 @@ protected boolean sendProxyBroadcastEnvelopeNow(String server, JsonEnvelope enve // Standalone broadcasts use the same initialized client as normal // envelopes. This preserves the socket connection and its delivery // acknowledgement instead of creating a second short-lived socket. - ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); - if (socketClient == null) { - return false; - } - try { - socketClient.sendEnvelope(envelope); - return true; - } catch (RuntimeException e) { - debug(e.getMessage()); - return false; - } + return sendSocketEnvelope(server, envelope); default: return false; } @@ -1381,10 +1371,7 @@ public void sendMessage(String server, int delay, JsonEnvelope envelope) { sendRedisEnvelopeServer(server, envelope); break; case SOCKETS: - ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); - if (socketClient != null) { - socketClient.sendEnvelope(envelope); - } + sendSocketEnvelope(server, envelope); break; default: break; @@ -2326,9 +2313,7 @@ public void onDisable() { if (socketHandler != null) { socketHandler.closeConnection(); } - HashMap clients = clientHandles; - clientHandles = null; - stopSocketClients(clients); + closeSocketClients(); if (redisHandler != null) { redisHandler.close(); @@ -2483,7 +2468,7 @@ private void reloadRuntime(boolean restartControlServices) { } } - private void rebuildSocketClients() { + private synchronized void rebuildSocketClients() { HashMap rebuilt = new HashMap<>(); try { List blocked = getConfig().getBlockedServers(); @@ -2503,6 +2488,24 @@ private void rebuildSocketClients() { stopSocketClients(previous); } + private synchronized boolean sendSocketEnvelope(String server, JsonEnvelope envelope) { + ClientHandler socketClient = clientHandles == null ? null : clientHandles.get(server); + if (socketClient == null) return false; + try { + socketClient.sendEnvelope(envelope); + return true; + } catch (RuntimeException e) { + debug(e.getMessage()); + return false; + } + } + + private synchronized void closeSocketClients() { + HashMap clients = clientHandles; + clientHandles = null; + stopSocketClients(clients); + } + static void stopSocketClients(Map clients) { if (clients == null) return; for (ClientHandler client : clients.values()) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java index 752b20ebfb..7aa9e42ea2 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandler.java @@ -53,7 +53,7 @@ public MultiProxyHandler() { /** * Closes the multi-proxy handler. */ - public void close() { + public synchronized void close() { if (multiproxySocketHandler != null) { multiproxySocketHandler.closeConnection(); multiproxySocketHandler = null; @@ -61,6 +61,9 @@ public void close() { if (multiProxyRedis != null && !getMultiProxyRedisUseExistingConnection()) { multiProxyRedis.close(); } + multiProxyRedis = null; + stopSocketClients(multiproxyClientHandles); + multiproxyClientHandles = null; } /** @@ -242,7 +245,7 @@ public abstract void triggerVote(String player, String service, boolean realVote /** * Loads multi-proxy support. */ - public void loadMultiProxySupport() { + public synchronized void loadMultiProxySupport() { if (!getMultiProxySupportEnabled()) { return; } @@ -347,7 +350,7 @@ public void sendStatus() { * * @param envelope the envelope to send */ - public void sendMultiProxyEnvelope(JsonEnvelope envelope) { + public synchronized void sendMultiProxyEnvelope(JsonEnvelope envelope) { if (envelope == null) { return; } @@ -368,6 +371,18 @@ public void sendMultiProxyEnvelope(JsonEnvelope envelope) { } } + static void stopSocketClients(Map clients) { + if (clients == null) return; + for (ClientHandler client : clients.values()) { + if (client == null) continue; + try { + client.stopConnection(); + } catch (RuntimeException ignored) { + // Continue closing the remaining clients after an individual failure. + } + } + } + private void handleEnvelope(JsonEnvelope envelope) { if (envelope == null) { return; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index ee9b2ba0db..227ceb281e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -4,12 +4,20 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.Test; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; +import com.bencodez.votingplugin.tests.VotingPluginProxyTestImpl; class VotingPluginProxyLifecycleTest { @@ -27,4 +35,52 @@ void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { verify(failing).stopConnection(); verify(healthy).stopConnection(); } + + @Test + void waitsForInFlightSocketSendBeforeClosingClientMap() throws Exception { + VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + proxy.setMethod(BungeeMethod.SOCKETS); + ClientHandler client = mock(ClientHandler.class); + JsonEnvelope envelope = mock(JsonEnvelope.class); + CountDownLatch sending = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch closingStarted = new CountDownLatch(1); + org.mockito.Mockito.doAnswer(ignored -> { + sending.countDown(); + if (!release.await(5, TimeUnit.SECONDS)) throw new AssertionError("send was not released"); + return null; + }).when(client).sendEnvelope(envelope); + + Field handles = VotingPluginProxy.class.getDeclaredField("clientHandles"); + handles.setAccessible(true); + HashMap clients = new HashMap<>(); + clients.put("lobby", client); + handles.set(proxy, clients); + Method close = VotingPluginProxy.class.getDeclaredMethod("closeSocketClients"); + close.setAccessible(true); + + var executor = Executors.newFixedThreadPool(2); + try { + var send = executor.submit(() -> proxy.sendProxyBroadcastEnvelopeNow("lobby", envelope)); + org.junit.jupiter.api.Assertions.assertTrue(sending.await(5, TimeUnit.SECONDS)); + var closing = executor.submit(() -> { + closingStarted.countDown(); + try { + close.invoke(proxy); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + }); + org.junit.jupiter.api.Assertions.assertTrue(closingStarted.await(5, TimeUnit.SECONDS)); + org.junit.jupiter.api.Assertions.assertThrows(java.util.concurrent.TimeoutException.class, + () -> closing.get(100, TimeUnit.MILLISECONDS)); + release.countDown(); + org.junit.jupiter.api.Assertions.assertTrue(send.get(5, TimeUnit.SECONDS)); + closing.get(5, TimeUnit.SECONDS); + verify(client).stopConnection(); + } finally { + release.countDown(); + executor.shutdownNow(); + } + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java new file mode 100644 index 0000000000..d1fbadaf97 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/multiproxy/MultiProxyHandlerLifecycleTest.java @@ -0,0 +1,30 @@ +package com.bencodez.votingplugin.proxy.multiproxy; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.LinkedHashMap; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; + +class MultiProxyHandlerLifecycleTest { + + @Test + void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { + ClientHandler failing = mock(ClientHandler.class); + ClientHandler healthy = mock(ClientHandler.class); + doThrow(new IllegalStateException("already closed")).when(failing).stopConnection(); + Map clients = new LinkedHashMap<>(); + clients.put("failing", failing); + clients.put("healthy", healthy); + + MultiProxyHandler.stopSocketClients(clients); + + verify(failing).stopConnection(); + verify(healthy).stopConnection(); + } +} From 2505c1422e78a294b06f05258c0c2929594c660b Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 19:16:42 -0600 Subject: [PATCH 19/73] Bound control transport activation --- .../backendproxy/BackendProxyHandler.java | 25 +++++++++ .../votingplugin/proxy/VotingPluginProxy.java | 2 +- .../proxy/control/HostedControlManager.java | 53 ++++++++++++++++--- .../control/HostedControlManagerTest.java | 41 +++++++++++++- 4 files changed, 112 insertions(+), 9 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index b032a47db7..89e702d11d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -50,6 +50,10 @@ import com.bencodez.votingplugin.util.ServiceSiteValidator; import com.bencodez.votingplugin.votesites.VoteSite; +import redis.clients.jedis.DefaultJedisClientConfig; +import redis.clients.jedis.HostAndPort; +import redis.clients.jedis.Jedis; + import lombok.Getter; /** @@ -592,6 +596,9 @@ public void validateTransport() { if (method.equals(BungeeMethod.REDIS) && redisHandler == null) { throw new IllegalStateException("Redis backend proxy transport initialization failed"); } + if (method.equals(BungeeMethod.REDIS)) { + validateRedisConnection(); + } if (method.equals(BungeeMethod.SOCKETS) && (clientHandler == null || socketHandler == null)) { throw new IllegalStateException("Socket backend proxy transport initialization failed"); } @@ -600,6 +607,24 @@ public void validateTransport() { } } + private void validateRedisConnection() { + DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder() + .database(plugin.getBungeeSettings().getRedisdbindex()) + .connectionTimeoutMillis(2000).socketTimeoutMillis(2000); + String username = plugin.getBungeeSettings().getRedisUsername(); + String password = plugin.getBungeeSettings().getRedisPassword(); + if (username != null && !username.isEmpty()) config.user(username); + if (password != null && !password.isEmpty()) config.password(password); + try (Jedis jedis = new Jedis(new HostAndPort(plugin.getBungeeSettings().getRedisHost(), + plugin.getBungeeSettings().getRedisPort()), config.build())) { + if (!"PONG".equalsIgnoreCase(jedis.ping())) { + throw new IllegalStateException("Redis backend proxy transport did not answer PING"); + } + } catch (RuntimeException failure) { + throw new IllegalStateException("Redis backend proxy transport connection failed", failure); + } + } + /** * Announces a player login to the configured proxy transport. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 576235159e..ea2f5679f1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2462,8 +2462,8 @@ private void reloadRuntime(boolean restartControlServices) { setCurrentVotePartyVotesRequired( getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); - loadMultiProxySupport(); if (restartControlServices) { + loadMultiProxySupport(); startControlServices(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 86778119b8..6f3b21d9df 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -499,16 +499,18 @@ interface ManagedProcess { CompletableFuture onExit(); } - private static final class JdkArtifactDownloader implements ArtifactDownloader { + static final class JdkArtifactDownloader implements ArtifactDownloader { private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)) .followRedirects(HttpClient.Redirect.NEVER).build(); @Override public void download(URI source, Path target, long maximumBytes, Duration timeout) throws IOException, InterruptedException { + long deadline = System.nanoTime() + timeout.toNanos(); URI current = source; for (int redirects = 0; redirects <= 5; redirects++) { - HttpRequest request = HttpRequest.newBuilder(current).timeout(timeout) + long requestNanos = remainingDownloadNanos(deadline); + HttpRequest request = HttpRequest.newBuilder(current).timeout(Duration.ofNanos(requestNanos)) .header("Accept", "application/java-archive, application/octet-stream") .header("User-Agent", "VotingPlugin-Control-Host").GET().build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); @@ -536,21 +538,58 @@ public void download(URI source, Path target, long maximumBytes, Duration timeou } try (InputStream input = response.body(); OutputStream output = Files.newOutputStream(target, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { + AtomicBoolean bodyTimedOut = new AtomicBoolean(); + Thread timeoutThread = bodyTimeoutThread(input, deadline, bodyTimedOut); byte[] buffer = new byte[8192]; long total = 0; int count; - while ((count = input.read(buffer)) != -1) { - total += count; - if (total > maximumBytes) { - throw new IOException("Control artifact exceeds the download limit"); + try { + while ((count = input.read(buffer)) != -1) { + if (bodyTimedOut.get() || System.nanoTime() >= deadline) { + throw new IOException("Control artifact download timed out"); + } + total += count; + if (total > maximumBytes) { + throw new IOException("Control artifact exceeds the download limit"); + } + output.write(buffer, 0, count); } - output.write(buffer, 0, count); + } catch (IOException failure) { + if (bodyTimedOut.get()) throw new IOException("Control artifact download timed out", failure); + throw failure; + } finally { + timeoutThread.interrupt(); } + if (bodyTimedOut.get()) throw new IOException("Control artifact download timed out"); } return; } throw new IOException("Control download redirected too many times"); } + + private static long remainingDownloadNanos(long deadline) throws IOException { + long remaining = deadline - System.nanoTime(); + if (remaining <= 0) throw new IOException("Control artifact download timed out"); + return remaining; + } + + private static Thread bodyTimeoutThread(InputStream input, long deadline, AtomicBoolean timedOut) + throws IOException { + long remaining = remainingDownloadNanos(deadline); + Thread thread = new Thread(() -> { + try { + TimeUnit.NANOSECONDS.sleep(remaining); + timedOut.set(true); + input.close(); + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } catch (IOException ignored) { } + }, "VotingPlugin-Control-Download-Timeout"); + thread.setDaemon(true); + thread.setContextClassLoader(ClassLoader.getPlatformClassLoader()); + thread.start(); + return thread; + } } private static final class JdkProcessLauncher implements ProcessLauncher { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index a3d4c017c5..19902485b1 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -5,23 +5,28 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; +import java.net.InetSocketAddress; import java.net.URI; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.time.Duration; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.CountDownLatch; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import com.sun.net.httpserver.HttpServer; + class HostedControlManagerTest { @TempDir Path directory; @@ -38,6 +43,40 @@ void versionPinsAndContainedPathsAreRequired() throws Exception { true, false, "not-a-digest", 30)); } + @Test + void hostedArtifactBodyReadCannotOutliveDownloadTimeout() throws Exception { + CountDownLatch bodyStarted = new CountDownLatch(1); + CountDownLatch releaseBody = new CountDownLatch(1); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/control.jar", exchange -> { + exchange.sendResponseHeaders(200, 0); + try (var body = exchange.getResponseBody()) { + body.write(1); + body.flush(); + bodyStarted.countDown(); + try { + releaseBody.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }); + server.start(); + try { + Path target = directory.resolve("stalled.jar"); + URI source = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/control.jar"); + long started = System.nanoTime(); + IOException failure = assertThrows(IOException.class, () -> new HostedControlManager.JdkArtifactDownloader() + .download(source, target, 1024, Duration.ofMillis(250))); + assertTrue(bodyStarted.await(2, TimeUnit.SECONDS)); + assertTrue(failure.getMessage().contains("timed out")); + assertTrue(Duration.ofNanos(System.nanoTime() - started).compareTo(Duration.ofSeconds(2)) < 0); + } finally { + releaseBody.countDown(); + server.stop(0); + } + } + @Test void missingPinnedArtifactIsStagedVerifiedAndStarted() throws Exception { byte[] release = "signed-release-content".getBytes(StandardCharsets.UTF_8); From 71d9acac023e478071e7ce5b45e54b3a723c3602 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 19:29:06 -0600 Subject: [PATCH 20/73] Bound connector responses and socket handoff --- .../votingplugin/VotingPluginMain.java | 6 ++ .../backendproxy/BackendProxyHandler.java | 8 ++ .../control/BackendControlConnector.java | 10 +-- .../proxy/control/ControlConnector.java | 19 +---- .../proxy/control/HostedControlManager.java | 31 +++----- .../util/BoundedHttpBodyHandler.java | 79 +++++++++++++++++++ .../BackendProxyHandlerLifecycleTest.java | 28 +++++++ .../control/HostedControlManagerTest.java | 31 ++++++++ 8 files changed, 172 insertions(+), 40 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/util/BoundedHttpBodyHandler.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index b1f9499344..d34fc6a778 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -66,6 +66,7 @@ import com.bencodez.votingplugin.broadcast.BroadcastHandler; import com.bencodez.votingplugin.backendproxy.BackendProxyHandler; import com.bencodez.votingplugin.broadcast.BroadcastSettings; +import com.bencodez.votingplugin.proxy.BungeeMethod; import com.bencodez.votingplugin.commands.CommandLoader; import com.bencodez.votingplugin.commands.executers.CommandAdminVote; import com.bencodez.votingplugin.commands.executers.CommandVote; @@ -1614,6 +1615,11 @@ public synchronized void restartBackendProxyHandler() { if (previous != null) previous.close(); return; } + BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); + if (previous != null && previous.getMethod() == BungeeMethod.SOCKETS + && replacementMethod == BungeeMethod.SOCKETS) { + previous.releaseSocketListener(); + } BackendProxyHandler replacement = new BackendProxyHandler(this); try { replacement.load(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 89e702d11d..2aa61c369a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -244,6 +244,14 @@ public void close() { } } + /** Releases the listening port before a same-address socket replacement is created. */ + public synchronized void releaseSocketListener() { + if (socketHandler != null) { + socketHandler.closeConnection(); + socketHandler = null; + } + } + /** * Loads and initializes the bungee handler with the configured method. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index a0bd1f36a6..0a68189374 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -1,7 +1,6 @@ package com.bencodez.votingplugin.control; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -26,6 +25,7 @@ import org.bukkit.configuration.ConfigurationSection; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; import com.bencodez.votingplugin.util.ControlCredentialFile; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -262,12 +262,8 @@ private Response send(String method, String path, JsonObject body) throws Except .timeout(Duration.ofMillis(settings.requestTimeoutMillis())).header("Content-Type", "application/json") .header("Authorization", "Bearer " + credential) .method(method, HttpRequest.BodyPublishers.ofString(body.toString(), StandardCharsets.UTF_8)).build(); - HttpResponse response = http.send(request, HttpResponse.BodyHandlers.ofInputStream()); - try (InputStream input = response.body()) { - byte[] bytes = input.readNBytes(MAX_RESPONSE_BYTES + 1); - if (bytes.length > MAX_RESPONSE_BYTES) throw new ConnectorException("response too large"); - return new Response(response.statusCode(), new String(bytes, StandardCharsets.UTF_8)); - } + HttpResponse response = http.send(request, new BoundedHttpBodyHandler(MAX_RESPONSE_BYTES)); + return new Response(response.statusCode(), new String(response.body(), StandardCharsets.UTF_8)); } private static JsonObject requireObject(Response response, int... statuses) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index e8065496f3..1728a67c6a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -1,7 +1,6 @@ package com.bencodez.votingplugin.proxy.control; import java.io.IOException; -import java.io.InputStream; import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; @@ -35,6 +34,7 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus; +import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; import com.bencodez.votingplugin.util.ControlCredentialFile; import com.google.gson.JsonArray; import com.google.gson.JsonElement; @@ -592,20 +592,9 @@ public CompletableFuture send(Request request) { .header("Authorization", "Bearer " + credential) .method(request.method, HttpRequest.BodyPublishers.ofString(request.body, StandardCharsets.UTF_8)) .build(); - return client.sendAsync(httpRequest, HttpResponse.BodyHandlers.ofInputStream()) - .thenApply(response -> new Response(response.statusCode(), readBounded(response.body()))); - } - - private static String readBounded(InputStream input) { - try (input) { - byte[] bytes = input.readNBytes(MAX_RESPONSE_BYTES + 1); - if (bytes.length > MAX_RESPONSE_BYTES) { - throw new MalformedResponseException(); - } - return new String(bytes, StandardCharsets.UTF_8); - } catch (IOException e) { - throw new CompletionException(new UnavailableException()); - } + return client.sendAsync(httpRequest, new BoundedHttpBodyHandler(MAX_RESPONSE_BYTES)) + .thenApply(response -> new Response(response.statusCode(), + new String(response.body(), StandardCharsets.UTF_8))); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 6f3b21d9df..5ea8d3af29 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -33,6 +33,7 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; +import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -627,7 +628,7 @@ private static boolean isWindows() { } } - private static final class JdkHealthProbe implements HealthProbe { + static final class JdkHealthProbe implements HealthProbe { private final HttpClient client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(2)) .followRedirects(HttpClient.Redirect.NEVER).build(); @@ -636,23 +637,17 @@ public boolean isHealthy(URI endpoint, Duration timeout) { try { HttpRequest request = HttpRequest.newBuilder(endpoint.resolve("/api/v1/health")) .timeout(timeout).header("Accept", "application/json").GET().build(); - HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofInputStream()); - try (InputStream input = response.body()) { - byte[] bytes = input.readNBytes(64 * 1024 + 1); - if (response.statusCode() != 200 || bytes.length > 64 * 1024) { - return false; - } - JsonElement parsed = JsonParser.parseString(new String(bytes, java.nio.charset.StandardCharsets.UTF_8)); - if (!parsed.isJsonObject()) { - return false; - } - JsonObject object = parsed.getAsJsonObject(); - JsonObject identity = object.has("identity") && object.get("identity").isJsonObject() - ? object.getAsJsonObject("identity") : null; - return object.has("status") && "ok".equals(object.get("status").getAsString()) - && identity != null && identity.has("protocolVersion") - && identity.get("protocolVersion").getAsInt() == EXPECTED_PROTOCOL_VERSION; - } + HttpResponse response = client.send(request, new BoundedHttpBodyHandler(64 * 1024)); + if (response.statusCode() != 200) return false; + JsonElement parsed = JsonParser.parseString( + new String(response.body(), java.nio.charset.StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) return false; + JsonObject object = parsed.getAsJsonObject(); + JsonObject identity = object.has("identity") && object.get("identity").isJsonObject() + ? object.getAsJsonObject("identity") : null; + return object.has("status") && "ok".equals(object.get("status").getAsString()) + && identity != null && identity.has("protocolVersion") + && identity.get("protocolVersion").getAsInt() == EXPECTED_PROTOCOL_VERSION; } catch (Exception e) { return false; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/BoundedHttpBodyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/BoundedHttpBodyHandler.java new file mode 100644 index 0000000000..eb9a732de3 --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/BoundedHttpBodyHandler.java @@ -0,0 +1,79 @@ +package com.bencodez.votingplugin.util; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Flow; + +/** Completes only after the entire bounded HTTP body has been consumed. */ +public final class BoundedHttpBodyHandler implements HttpResponse.BodyHandler { + private final int maximumBytes; + + public BoundedHttpBodyHandler(int maximumBytes) { + if (maximumBytes < 0) throw new IllegalArgumentException("maximumBytes must be non-negative"); + this.maximumBytes = maximumBytes; + } + + @Override + public HttpResponse.BodySubscriber apply(HttpResponse.ResponseInfo responseInfo) { + return new Subscriber(maximumBytes); + } + + private static final class Subscriber implements HttpResponse.BodySubscriber { + private final int maximumBytes; + private final ByteArrayOutputStream bytes; + private final CompletableFuture result = new CompletableFuture<>(); + private Flow.Subscription subscription; + + private Subscriber(int maximumBytes) { + this.maximumBytes = maximumBytes; + this.bytes = new ByteArrayOutputStream(Math.min(maximumBytes, 8192)); + } + + @Override + public CompletionStage getBody() { + return result; + } + + @Override + public void onSubscribe(Flow.Subscription selected) { + if (subscription != null) { + selected.cancel(); + return; + } + subscription = selected; + selected.request(1); + } + + @Override + public void onNext(List items) { + if (result.isDone()) return; + for (ByteBuffer item : items) { + int count = item.remaining(); + if (count > maximumBytes - bytes.size()) { + subscription.cancel(); + result.completeExceptionally(new IOException("HTTP response body exceeds the limit")); + return; + } + byte[] chunk = new byte[count]; + item.get(chunk); + bytes.write(chunk, 0, chunk.length); + } + subscription.request(1); + } + + @Override + public void onError(Throwable failure) { + result.completeExceptionally(failure); + } + + @Override + public void onComplete() { + result.complete(bytes.toByteArray()); + } + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java new file mode 100644 index 0000000000..0d4624e4fc --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -0,0 +1,28 @@ +package com.bencodez.votingplugin.backendproxy; + +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.lang.reflect.Field; + +import org.junit.jupiter.api.Test; + +import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; + +class BackendProxyHandlerLifecycleTest { + + @Test + void releasesSocketListenerBeforeSamePortReplacement() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + SocketHandler socket = mock(SocketHandler.class); + Field field = BackendProxyHandler.class.getDeclaredField("socketHandler"); + field.setAccessible(true); + field.set(handler, socket); + + handler.releaseSocketListener(); + + verify(socket).closeConnection(); + assertNull(handler.getSocketHandler()); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 19902485b1..7934ba893f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -77,6 +77,37 @@ void hostedArtifactBodyReadCannotOutliveDownloadTimeout() throws Exception { } } + @Test + void hostedHealthBodyReadCannotOutliveProbeTimeout() throws Exception { + CountDownLatch bodyStarted = new CountDownLatch(1); + CountDownLatch releaseBody = new CountDownLatch(1); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/api/v1/health", exchange -> { + exchange.sendResponseHeaders(200, 0); + try (var body = exchange.getResponseBody()) { + body.write("{\"status\":\"ok\"".getBytes(StandardCharsets.UTF_8)); + body.flush(); + bodyStarted.countDown(); + try { + releaseBody.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + }); + server.start(); + try { + URI endpoint = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/"); + long started = System.nanoTime(); + assertFalse(new HostedControlManager.JdkHealthProbe().isHealthy(endpoint, Duration.ofMillis(250))); + assertTrue(bodyStarted.await(2, TimeUnit.SECONDS)); + assertTrue(Duration.ofNanos(System.nanoTime() - started).compareTo(Duration.ofSeconds(2)) < 0); + } finally { + releaseBody.countDown(); + server.stop(0); + } + } + @Test void missingPinnedArtifactIsStagedVerifiedAndStarted() throws Exception { byte[] release = "signed-release-content".getBytes(StandardCharsets.UTF_8); From 4f251b141f9a18a30943870863a97e2a10127142 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 19:33:12 -0600 Subject: [PATCH 21/73] Enforce connector body deadlines --- .../control/BackendControlConnector.java | 4 ++- .../proxy/control/ControlConnector.java | 2 +- .../proxy/control/HostedControlManager.java | 2 +- .../util/BoundedHttpBodyHandler.java | 30 +++++++++++++++++-- 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 0a68189374..6e3a646fb3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -262,7 +262,9 @@ private Response send(String method, String path, JsonObject body) throws Except .timeout(Duration.ofMillis(settings.requestTimeoutMillis())).header("Content-Type", "application/json") .header("Authorization", "Bearer " + credential) .method(method, HttpRequest.BodyPublishers.ofString(body.toString(), StandardCharsets.UTF_8)).build(); - HttpResponse response = http.send(request, new BoundedHttpBodyHandler(MAX_RESPONSE_BYTES)); + HttpResponse response = http.send(request, + new BoundedHttpBodyHandler(MAX_RESPONSE_BYTES, + Duration.ofMillis(settings.requestTimeoutMillis()))); return new Response(response.statusCode(), new String(response.body(), StandardCharsets.UTF_8)); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 1728a67c6a..ca2242be4b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -592,7 +592,7 @@ public CompletableFuture send(Request request) { .header("Authorization", "Bearer " + credential) .method(request.method, HttpRequest.BodyPublishers.ofString(request.body, StandardCharsets.UTF_8)) .build(); - return client.sendAsync(httpRequest, new BoundedHttpBodyHandler(MAX_RESPONSE_BYTES)) + return client.sendAsync(httpRequest, new BoundedHttpBodyHandler(MAX_RESPONSE_BYTES, requestTimeout)) .thenApply(response -> new Response(response.statusCode(), new String(response.body(), StandardCharsets.UTF_8))); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 5ea8d3af29..1998e4ccc0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -637,7 +637,7 @@ public boolean isHealthy(URI endpoint, Duration timeout) { try { HttpRequest request = HttpRequest.newBuilder(endpoint.resolve("/api/v1/health")) .timeout(timeout).header("Accept", "application/json").GET().build(); - HttpResponse response = client.send(request, new BoundedHttpBodyHandler(64 * 1024)); + HttpResponse response = client.send(request, new BoundedHttpBodyHandler(64 * 1024, timeout)); if (response.statusCode() != 200) return false; JsonElement parsed = JsonParser.parseString( new String(response.body(), java.nio.charset.StandardCharsets.UTF_8)); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/BoundedHttpBodyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/BoundedHttpBodyHandler.java index eb9a732de3..218819c9dc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/BoundedHttpBodyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/BoundedHttpBodyHandler.java @@ -2,36 +2,45 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.net.http.HttpTimeoutException; import java.net.http.HttpResponse; import java.nio.ByteBuffer; +import java.time.Duration; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; import java.util.concurrent.Flow; +import java.util.concurrent.TimeUnit; /** Completes only after the entire bounded HTTP body has been consumed. */ public final class BoundedHttpBodyHandler implements HttpResponse.BodyHandler { private final int maximumBytes; + private final Duration timeout; - public BoundedHttpBodyHandler(int maximumBytes) { + public BoundedHttpBodyHandler(int maximumBytes, Duration timeout) { if (maximumBytes < 0) throw new IllegalArgumentException("maximumBytes must be non-negative"); + if (timeout == null || timeout.isNegative() || timeout.isZero()) + throw new IllegalArgumentException("timeout must be positive"); this.maximumBytes = maximumBytes; + this.timeout = timeout; } @Override public HttpResponse.BodySubscriber apply(HttpResponse.ResponseInfo responseInfo) { - return new Subscriber(maximumBytes); + return new Subscriber(maximumBytes, timeout); } private static final class Subscriber implements HttpResponse.BodySubscriber { private final int maximumBytes; private final ByteArrayOutputStream bytes; private final CompletableFuture result = new CompletableFuture<>(); + private final long timeoutNanos; private Flow.Subscription subscription; - private Subscriber(int maximumBytes) { + private Subscriber(int maximumBytes, Duration timeout) { this.maximumBytes = maximumBytes; this.bytes = new ByteArrayOutputStream(Math.min(maximumBytes, 8192)); + this.timeoutNanos = timeout.toNanos(); } @Override @@ -46,6 +55,21 @@ public void onSubscribe(Flow.Subscription selected) { return; } subscription = selected; + Thread timeoutThread = new Thread(() -> { + try { + TimeUnit.NANOSECONDS.sleep(timeoutNanos); + if (!result.isDone()) { + selected.cancel(); + result.completeExceptionally(new HttpTimeoutException("HTTP response body timed out")); + } + } catch (InterruptedException ignored) { + Thread.currentThread().interrupt(); + } + }, "VotingPlugin-Control-Response-Timeout"); + timeoutThread.setDaemon(true); + timeoutThread.setContextClassLoader(ClassLoader.getPlatformClassLoader()); + result.whenComplete((ignored, failure) -> timeoutThread.interrupt()); + timeoutThread.start(); selected.request(1); } From 6804cad4c01cfe5b5621fe87f1b1525d4b9d513d Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 19:37:23 -0600 Subject: [PATCH 22/73] Trigger verification for response deadlines From 2bd8507445eb4dc0b53bd783451cfbbed2cea853 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 19:46:43 -0600 Subject: [PATCH 23/73] Serialize Redis backend replacement --- .../bencodez/votingplugin/VotingPluginMain.java | 4 ++++ .../backendproxy/BackendProxyHandler.java | 12 ++++++++++++ .../BackendProxyHandlerLifecycleTest.java | 15 +++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index d34fc6a778..6b67552cc4 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1620,6 +1620,10 @@ public synchronized void restartBackendProxyHandler() { && replacementMethod == BungeeMethod.SOCKETS) { previous.releaseSocketListener(); } + if (previous != null && previous.getMethod() == BungeeMethod.REDIS + && replacementMethod == BungeeMethod.REDIS) { + previous.releaseRedisTransport(); + } BackendProxyHandler replacement = new BackendProxyHandler(this); try { replacement.load(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 2aa61c369a..f833657d6e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -252,6 +252,18 @@ public synchronized void releaseSocketListener() { } } + /** Stops a shared-topic subscriber before a Redis replacement is activated. */ + public synchronized void releaseRedisTransport() { + if (redisHandler != null) { + redisHandler.close(); + redisHandler = null; + } + if (redisThread != null) { + redisThread.interrupt(); + redisThread = null; + } + } + /** * Loads and initializes the bungee handler with the configured method. */ diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 0d4624e4fc..b836856176 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; +import com.bencodez.simpleapi.servercomm.redis.RedisHandler; class BackendProxyHandlerLifecycleTest { @@ -25,4 +26,18 @@ void releasesSocketListenerBeforeSamePortReplacement() throws Exception { verify(socket).closeConnection(); assertNull(handler.getSocketHandler()); } + + @Test + void releasesRedisSubscriberBeforeSameTopicReplacement() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + RedisHandler redis = mock(RedisHandler.class); + Field field = BackendProxyHandler.class.getDeclaredField("redisHandler"); + field.setAccessible(true); + field.set(handler, redis); + + handler.releaseRedisTransport(); + + verify(redis).close(); + assertNull(handler.getRedisHandler()); + } } From 0a48fceb4a50e60ad0d0478ff4b12b598a7fdcca Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 19:57:10 -0600 Subject: [PATCH 24/73] Wait for hosted runtime handoff --- .../bencodez/votingplugin/proxy/VotingPluginProxy.java | 7 ++++++- .../bencodez/votingplugin/proxy/bungee/BungeeConfig.java | 3 +++ .../votingplugin/proxy/bungee/VotingPluginBungee.java | 8 ++++---- .../votingplugin/proxy/velocity/VelocityConfig.java | 3 +++ .../votingplugin/proxy/velocity/VotingPluginVelocity.java | 2 +- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index ea2f5679f1..955ab717a8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2296,7 +2296,12 @@ private void logInfo(String msg) { public abstract void logSevere(String message); public void onDisable() { - stopControlServices(false); + onDisable(false); + } + + /** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */ + public void onDisable(boolean waitForHosted) { + stopControlServices(waitForHosted); getVoteCacheHandler().saveVoteCache(); if (getProxyMysqlMessenger() != null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index ef834fcfbe..dbd8cf9a99 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -539,6 +539,9 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer throw new StaleControlRevisionException(); } atomicReplace(backupStage, backup); + if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } atomicReplace(stage, target); data = latest; } finally { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index 9c729f0436..d5b5c98e2c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -218,8 +218,8 @@ public void onEnable() { } } - @Override - public void onDisable() { + @Override + public void onDisable() { synchronized (reloadLock) { reloading = true; @@ -227,7 +227,7 @@ public void onDisable() { try { if (votingPluginProxy != null) { - votingPluginProxy.onDisable(); + votingPluginProxy.onDisable(); } } catch (Exception ignored) { } @@ -396,7 +396,7 @@ public void reloadPlugin(boolean loadMysql) { // Tear down old runtime (this WILL shutdown MySQL because proxy owns it) try { if (votingPluginProxy != null) { - votingPluginProxy.onDisable(); + votingPluginProxy.onDisable(true); } } catch (Exception ignored) { } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index f24f322cdf..e2b868145a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -63,6 +63,9 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer throw new StaleControlRevisionException(); } atomicReplace(backupStage, backup); + if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } atomicReplace(stage, target); } finally { Files.deleteIfExists(stage); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 6d6a613ac6..e8a55b4e68 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -481,7 +481,7 @@ public void reloadAllInternal(boolean loadMysql) { // Shutdown old runtime completely (this stops old MySQL pool) try { if (votingPluginProxy != null) { - votingPluginProxy.onDisable(); + votingPluginProxy.onDisable(true); } } catch (Exception ignored) { } From db3cfc027b36e61e961cb4b9ced0dd1f03a57fe0 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 20:08:39 -0600 Subject: [PATCH 25/73] Abort unsafe reloads and protect concurrent edits --- .../control/BackendConfigurationService.java | 4 ++++ .../proxy/bungee/VotingPluginBungee.java | 19 ++++++++++++----- .../proxy/velocity/VotingPluginVelocity.java | 10 ++++++++- .../BackendConfigurationServiceTest.java | 21 +++++++++++++++++++ 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 5573deb9df..ac55d39699 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -82,6 +82,10 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte boolean rolledBack = false; if (installed) { try { + String installedRevision = revision(preview.resolvedContent()); + if (!revision(readRaw(target, false)).equals(installedRevision)) { + throw new IOException("Managed configuration changed while reload failed; backup was not restored"); + } Files.copy(backup, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); reload.run(fileName); rolledBack = true; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index d5b5c98e2c..784cccd587 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -394,12 +394,21 @@ public void reloadPlugin(boolean loadMysql) { } // Tear down old runtime (this WILL shutdown MySQL because proxy owns it) - try { - if (votingPluginProxy != null) { + try { + if (votingPluginProxy != null) { votingPluginProxy.onDisable(true); - } - } catch (Exception ignored) { - } + } + } catch (Exception shutdownFailure) { + getLogger().severe("Reload aborted because hosted Control did not stop safely"); + shutdownFailure.printStackTrace(); + try { + schedulePlatformTasks(); + } catch (Exception taskFailure) { + shutdownFailure.addSuppressed(taskFailure); + } + reloading = false; + return; + } // Recreate runtime votingPluginProxy = createProxyRuntime(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index e8a55b4e68..235b50a5af 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -483,7 +483,15 @@ public void reloadAllInternal(boolean loadMysql) { if (votingPluginProxy != null) { votingPluginProxy.onDisable(true); } - } catch (Exception ignored) { + } catch (Exception shutdownFailure) { + logger.error("Reload aborted because hosted Control did not stop safely", shutdownFailure); + try { + scheduleTasks(); + } catch (Exception taskFailure) { + shutdownFailure.addSuppressed(taskFailure); + } + reloading = false; + return; } // Recreate runtime diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 10927de89b..ea3ce5a263 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -86,6 +86,27 @@ class BackendConfigurationServiceTest { assertTrue(service.read("BungeeSettings.yml").content().contains("PLUGINMESSAGING")); } + @Test void failedReloadDoesNotOverwriteAConcurrentManualEdit() throws Exception { + Path config = directory.resolve("Config.yml"); + Files.writeString(config, "Feature: before\n"); + AtomicInteger reloads = new AtomicInteger(); + BackendConfigurationService service = new BackendConfigurationService(directory, + (BackendConfigurationService.ApplyAction) fileName -> { + reloads.incrementAndGet(); + Files.writeString(config, "Feature: manual\n"); + throw new IllegalStateException("reload failed"); + }); + BackendConfigurationService.Document before = service.read("Config.yml"); + + BackendConfigurationService.ApplyFailureException failure = assertThrows( + BackendConfigurationService.ApplyFailureException.class, + () -> service.apply("Config.yml", "Feature: proposed\n", before.revision())); + + assertFalse(failure.rolledBack()); + assertEquals(1, reloads.get()); + assertEquals("Feature: manual\n", Files.readString(config)); + } + @Test void quickSetupsProduceReviewableBackendAndVoteSiteChanges() throws Exception { Files.writeString(directory.resolve("BungeeSettings.yml"), "UseBungeecord: false\nServer: PleaseSet\n"); Files.writeString(directory.resolve("VoteSites.yml"), "VoteSites: {}\n"); From 04628fa43c599eff9a6dda2ed7780e0a785533b4 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 20:58:03 -0600 Subject: [PATCH 26/73] Protect proxy reload handoffs --- .../votingplugin/VotingPluginMain.java | 4 +++ .../backendproxy/BackendProxyHandler.java | 12 ++++++++ .../proxy/bungee/BungeeConfig.java | 10 ++++++- .../proxy/velocity/VelocityConfig.java | 8 +++++ .../BackendProxyHandlerLifecycleTest.java | 15 ++++++++++ .../ProxyRoutingConfigurationServiceTest.java | 30 ++++++++++++++++++- 6 files changed, 77 insertions(+), 2 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 6b67552cc4..ab2baf2f10 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1624,6 +1624,10 @@ public synchronized void restartBackendProxyHandler() { && replacementMethod == BungeeMethod.REDIS) { previous.releaseRedisTransport(); } + if (previous != null && previous.getMethod() == BungeeMethod.MQTT + && replacementMethod == BungeeMethod.MQTT) { + previous.releaseMqttTransport(); + } BackendProxyHandler replacement = new BackendProxyHandler(this); try { replacement.load(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index f833657d6e..82d02e2eb8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -264,6 +264,18 @@ public synchronized void releaseRedisTransport() { } } + /** Stops an MQTT subscriber before another client is connected for the same backend method. */ + public synchronized void releaseMqttTransport() { + if (mqttHandler != null) { + try { + mqttHandler.disconnect(); + } catch (Exception e) { + throw new IllegalStateException("Unable to disconnect the replaced MQTT backend transport", e); + } + mqttHandler = null; + } + } + /** * Loads and initializes the bungee handler with the configured method. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index dbd8cf9a99..6bad31a653 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -24,7 +24,8 @@ * Configuration implementation for Bungee proxy server. */ public class BungeeConfig implements VotingPluginProxyConfig { - private VotingPluginBungee bungee; + private VotingPluginBungee bungee; + private byte[] controlInstalledSnapshot; @Getter private Configuration data; @@ -531,6 +532,7 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer latest.set("SendVotesToAllServers", sendVotesToAllServers); latest.set("BlockedServers", List.copyOf(blockedServers)); ConfigurationProvider.getProvider(YamlConfiguration.class).save(latest, stage.toFile()); + byte[] installedSnapshot = Files.readAllBytes(stage); if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } @@ -543,6 +545,7 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer throw new StaleControlRevisionException(); } atomicReplace(stage, target); + controlInstalledSnapshot = installedSnapshot; data = latest; } finally { Files.deleteIfExists(stage); @@ -560,9 +563,14 @@ public synchronized void rollbackControlProxyRouting() throws IOException { Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); if (!Files.isRegularFile(backup)) throw new IOException("Control backup is unavailable"); + if (controlInstalledSnapshot == null + || !java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); atomicReplace(stage, target); + controlInstalledSnapshot = null; data = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index e2b868145a..73f1bd6907 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -26,6 +26,7 @@ */ public class VelocityConfig extends VelocityYMLFile implements VotingPluginProxyConfig { private final File configurationFile; + private byte[] controlInstalledSnapshot; /** * Constructs a new Velocity configuration. @@ -55,6 +56,7 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer latest.node("SendVotesToAllServers").set(sendVotesToAllServers); latest.node("BlockedServers").setList(String.class, List.copyOf(blockedServers)); YamlConfigurationLoader.builder().path(stage).build().save(latest); + byte[] installedSnapshot = Files.readAllBytes(stage); if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } @@ -67,6 +69,7 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer throw new StaleControlRevisionException(); } atomicReplace(stage, target); + controlInstalledSnapshot = installedSnapshot; } finally { Files.deleteIfExists(stage); if (backupStage != null) Files.deleteIfExists(backupStage); @@ -83,9 +86,14 @@ public synchronized void rollbackControlProxyRouting() throws IOException { Path target = configurationFile.toPath(); Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); if (!Files.isRegularFile(backup)) throw new IOException("Control backup is unavailable"); + if (controlInstalledSnapshot == null + || !java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); atomicReplace(stage, target); + controlInstalledSnapshot = null; loadControlConfiguration(); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index b836856176..85a132188a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -9,6 +9,7 @@ import org.junit.jupiter.api.Test; import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; +import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; class BackendProxyHandlerLifecycleTest { @@ -40,4 +41,18 @@ void releasesRedisSubscriberBeforeSameTopicReplacement() throws Exception { verify(redis).close(); assertNull(handler.getRedisHandler()); } + + @Test + void releasesMqttSubscriberBeforeSameMethodReplacement() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + MqttHandler mqtt = mock(MqttHandler.class); + Field field = BackendProxyHandler.class.getDeclaredField("mqttHandler"); + field.setAccessible(true); + field.set(handler, mqtt); + + handler.releaseMqttTransport(); + + verify(mqtt).disconnect(); + assertNull(handler.getMqttHandler()); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java index 6cd2d87477..b130474189 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; @@ -52,12 +53,31 @@ void persistenceRevisionRaceRemainsAStaleRevision() { assertTrue(platform.reloadCalls == 0); } + @Test + void failedReloadPreservesAConcurrentEditDetectedByTheAdapter() { + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + FakePlatform platform = new FakePlatform(); + platform.current = current; + platform.manualEditDuringFailedReload = true; + + ProxyRoutingConfigurationService.ApplyFailureException failure = assertThrows( + ProxyRoutingConfigurationService.ApplyFailureException.class, + () -> new ProxyRoutingConfigurationService(platform).apply( + new ProxyRoutingConfiguration(true, List.of()), current.revision())); + + assertFalse(failure.rolledBack()); + assertFalse(platform.rolledBack); + assertTrue(platform.current.blockedServers().contains("manual-edit")); + } + private static final class FakePlatform implements ProxyRoutingConfigurationService.Platform { private ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); private Set configuredServers = Set.of(); private boolean failFirstReload; private boolean rolledBack; private boolean staleDuringPersist; + private boolean manualEditDuringFailedReload; + private ProxyRoutingConfiguration installed; private int reloadCalls; @Override public ProxyRoutingConfiguration read() { return current; } @@ -65,10 +85,18 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) throws IOException { if (staleDuringPersist) throw new com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException(); current = proposal; + installed = proposal; + } + @Override public void rollback() throws IOException { + if (!current.equals(installed)) throw new IOException("active configuration changed"); + rolledBack = true; } - @Override public void rollback() { rolledBack = true; } @Override public void reload() throws Exception { reloadCalls++; + if (manualEditDuringFailedReload && reloadCalls == 1) { + current = new ProxyRoutingConfiguration(false, List.of("manual-edit")); + throw new IOException("invalid reloaded configuration"); + } if (failFirstReload && reloadCalls == 1) throw new IOException("invalid reloaded configuration"); } } From d80e562c5be8788b9990dbde75c50b3e2e1127df Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 21:09:27 -0600 Subject: [PATCH 27/73] Bound proxy control reload callbacks --- .../votingplugin/proxy/VotingPluginProxy.java | 35 ++++++++++++++++++- .../proxy/control/ControlConnector.java | 3 ++ .../proxy/control/ControlConnectorTest.java | 11 ++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 955ab717a8..4c3f8e4e79 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -27,8 +27,11 @@ import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import org.eclipse.paho.client.mqttv3.MqttException; @@ -165,6 +168,12 @@ public abstract class VotingPluginProxy { private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); private volatile ControlConnector controlConnector; private volatile HostedControlManager hostedControlManager; + private final AtomicLong controlServicesGeneration = new AtomicLong(); + private final ExecutorService controlLifecycleExecutor = Executors.newSingleThreadExecutor(task -> { + Thread thread = new Thread(task, "votingplugin-control-lifecycle"); + thread.setDaemon(true); + return thread; + }); public VotingPluginProxy() { enabled = true; @@ -1567,6 +1576,27 @@ private synchronized void startControlServices() { } } + /** Keeps potentially long hosted-Control handoffs off proxy command/event threads. */ + private void restartControlServicesAsync() { + final long generation = controlServicesGeneration.incrementAndGet(); + try { + controlLifecycleExecutor.execute(() -> { + try { + synchronized (VotingPluginProxy.this) { + if (!enabled || generation != controlServicesGeneration.get()) return; + startControlServices(); + } + } catch (RuntimeException failure) { + if (generation == controlServicesGeneration.get()) { + logSevere("[Control] asynchronous service restart failed: " + failure.getMessage()); + } + } + }); + } catch (RuntimeException failure) { + logSevere("[Control] services were not restarted because async scheduling failed"); + } + } + private synchronized void stopControlServices(boolean waitForHosted) { ControlConnector connector = controlConnector; controlConnector = null; @@ -2301,6 +2331,9 @@ public void onDisable() { /** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */ public void onDisable(boolean waitForHosted) { + controlServicesGeneration.incrementAndGet(); + if (waitForHosted) controlLifecycleExecutor.shutdown(); + else controlLifecycleExecutor.shutdownNow(); stopControlServices(waitForHosted); getVoteCacheHandler().saveVoteCache(); @@ -2469,7 +2502,7 @@ private void reloadRuntime(boolean restartControlServices) { getConfig().getVotePartyVotesRequired() + getVoteCacheVotePartyIncreaseVotesRequired()); if (restartControlServices) { loadMultiProxySupport(); - startControlServices(); + restartControlServicesAsync(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index ca2242be4b..4a25655232 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -220,6 +220,9 @@ private void handlePrimaryResponse(Response response, boolean registration) { } } if (accepted != null) { + if (!contains(accepted, "presence.snapshot")) { + throw new ProtocolException(); + } configurationAccepted = contains(accepted, CONFIGURATION_CAPABILITY); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 12b25af880..ef74e55980 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -93,6 +93,17 @@ class ControlConnectorTest { assertEquals("/api/v1/nodes/register", transport.requests.get(3).path()); } + @Test void explicitHeartbeatCapabilityRevocationIsIncompatibleAndStopsTheCycle() { + connector.cycle(); + transport.nextPrimary = new Response(200, + "{\"node\":{\"acceptedCapabilities\":[\"config.proxy-routing.v1\"]}}"); + + connector.cycle(); + + assertEquals(Status.INCOMPATIBLE, connector.status()); + assertEquals(3, transport.requests.size()); + } + @Test void skipsOperationClaimsWhenConfigurationCapabilityWasNotAccepted() { connector.close(); ProxyRoutingConfigurationService service = new ProxyRoutingConfigurationService(new NoOpPlatform()); From 4607ce992d336576a8157e5a66655f17ddefab70 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 21:18:48 -0600 Subject: [PATCH 28/73] Isolate control lifecycle and backup paths --- .../control/BackendConfigurationService.java | 31 +++++++++- .../votingplugin/proxy/VotingPluginProxy.java | 62 +++++++++++-------- .../BackendConfigurationServiceTest.java | 15 +++++ 3 files changed, 79 insertions(+), 29 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index ac55d39699..9dc5100130 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -6,9 +6,12 @@ import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; @@ -63,13 +66,16 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte Preview preview = preview(fileName, proposedContent, current); Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); Path staging = Files.createTempFile(target.getParent(), ".control-", ".yml"); + Path backupStaging = Files.createTempFile(target.getParent(), ".control-backup-", ".yml"); boolean installed = false; try { Files.writeString(staging, preview.resolvedContent(), StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING); - Files.writeString(backup, current, StandardCharsets.UTF_8, StandardOpenOption.CREATE, + rejectSymbolicBackup(backup); + Files.writeString(backupStaging, current, StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); + move(backupStaging, backup); move(staging, target); installed = true; reload.run(fileName); @@ -86,7 +92,7 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte if (!revision(readRaw(target, false)).equals(installedRevision)) { throw new IOException("Managed configuration changed while reload failed; backup was not restored"); } - Files.copy(backup, target, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.COPY_ATTRIBUTES); + copyBackupNoFollow(backup, target); reload.run(fileName); rolledBack = true; } catch (Exception rollbackFailure) { @@ -94,6 +100,27 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte } } throw new ApplyFailureException(rolledBack, failure); + } finally { + Files.deleteIfExists(staging); + Files.deleteIfExists(backupStaging); + } + } + + private static void rejectSymbolicBackup(Path backup) throws IOException { + if (Files.isSymbolicLink(backup)) throw new IOException("Symbolic configuration backups are not allowed"); + } + + private static void copyBackupNoFollow(Path backup, Path target) throws IOException { + if (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Control backup is unavailable or unsafe"); + } + Path staging = Files.createTempFile(target.getParent(), ".control-rollback-", ".yml"); + try { + try (SeekableByteChannel source = Files.newByteChannel(backup, + Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { + Files.copy(Channels.newInputStream(source), staging, StandardCopyOption.REPLACE_EXISTING); + } + move(staging, target); } finally { Files.deleteIfExists(staging); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 4c3f8e4e79..1d0fb5344d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -168,6 +168,7 @@ public abstract class VotingPluginProxy { private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); private volatile ControlConnector controlConnector; private volatile HostedControlManager hostedControlManager; + private final Object controlLifecycleLock = new Object(); private final AtomicLong controlServicesGeneration = new AtomicLong(); private final ExecutorService controlLifecycleExecutor = Executors.newSingleThreadExecutor(task -> { Thread thread = new Thread(task, "votingplugin-control-lifecycle"); @@ -1549,30 +1550,10 @@ public void onReceive(JsonEnvelope message) { debug("VotingPluginProxy loaded, ONLINEMODE: " + getConfig().getOnlineMode()); } - private synchronized void startControlServices() { - stopControlServices(true); - if (getConfig().getControlHostedEnabled()) { - try { - hostedControlManager = HostedControlManager.create(this); - if (hostedControlManager != null) { - hostedControlManager.start(); - } - } catch (IllegalArgumentException e) { - hostedControlManager = null; - logSevere("[Control Host] configuration is invalid; VotingPlugin remains unaffected"); - } - } - if (!getConfig().getControlEnabled()) { - return; - } - try { - controlConnector = ControlConnector.create(this); - if (controlConnector != null) { - controlConnector.start(); - } - } catch (IOException | IllegalArgumentException e) { - controlConnector = null; - logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); + private void startControlServices() { + synchronized (controlLifecycleLock) { + stopControlServicesLocked(true); + startControlServicesLocked(); } } @@ -1582,9 +1563,10 @@ private void restartControlServicesAsync() { try { controlLifecycleExecutor.execute(() -> { try { - synchronized (VotingPluginProxy.this) { + synchronized (controlLifecycleLock) { if (!enabled || generation != controlServicesGeneration.get()) return; - startControlServices(); + stopControlServicesLocked(true); + startControlServicesLocked(); } } catch (RuntimeException failure) { if (generation == controlServicesGeneration.get()) { @@ -1597,7 +1579,33 @@ private void restartControlServicesAsync() { } } - private synchronized void stopControlServices(boolean waitForHosted) { + private void stopControlServices(boolean waitForHosted) { + synchronized (controlLifecycleLock) { + stopControlServicesLocked(waitForHosted); + } + } + + private void startControlServicesLocked() { + if (getConfig().getControlHostedEnabled()) { + try { + hostedControlManager = HostedControlManager.create(this); + if (hostedControlManager != null) hostedControlManager.start(); + } catch (IllegalArgumentException e) { + hostedControlManager = null; + logSevere("[Control Host] configuration is invalid; VotingPlugin remains unaffected"); + } + } + if (!getConfig().getControlEnabled()) return; + try { + controlConnector = ControlConnector.create(this); + if (controlConnector != null) controlConnector.start(); + } catch (IOException | IllegalArgumentException e) { + controlConnector = null; + logSevere("[Control] connector configuration or credential is invalid; voting remains unaffected"); + } + } + + private void stopControlServicesLocked(boolean waitForHosted) { ControlConnector connector = controlConnector; controlConnector = null; if (connector != null) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index ea3ce5a263..86ee2408ca 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -66,6 +66,21 @@ class BackendConfigurationServiceTest { assertThrows(java.io.IOException.class, () -> service.read("VoteSites/External.yml")); } + @Test void rejectsSymlinkedBackupSidecarsWithoutTouchingTheirTargets() throws Exception { + Path config = directory.resolve("Config.yml"); + Path external = directory.resolve("external-backup.yml"); + Files.writeString(config, "Feature: false\n"); + Files.writeString(external, "external: preserved\n"); + Files.createSymbolicLink(directory.resolve("Config.yml.control-backup"), external); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + String revision = service.read("Config.yml").revision(); + + assertThrows(java.io.IOException.class, + () -> service.apply("Config.yml", "Feature: true\n", revision)); + assertEquals("external: preserved\n", Files.readString(external)); + assertEquals("Feature: false\n", Files.readString(config)); + } + @Test void transportValidationFailureRollsBackBeforeApplyReturns() throws Exception { Path settings = directory.resolve("BungeeSettings.yml"); Files.writeString(settings, "UseBungeecord: false\nBungeeMethod: PLUGINMESSAGING\n"); From d9b7a91854d4c204083c461dd9495edadd35e5b3 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 21:32:16 -0600 Subject: [PATCH 29/73] Close remaining backend handoff races --- .../bencodez/votingplugin/VotingPluginMain.java | 4 ++++ .../backendproxy/BackendProxyHandler.java | 8 ++++++++ .../control/BackendConfigurationService.java | 1 + .../BackendProxyHandlerLifecycleTest.java | 15 +++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index ab2baf2f10..f38d781fc1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1628,6 +1628,10 @@ public synchronized void restartBackendProxyHandler() { && replacementMethod == BungeeMethod.MQTT) { previous.releaseMqttTransport(); } + if (previous != null && previous.getMethod() == BungeeMethod.MYSQL + && replacementMethod == BungeeMethod.MYSQL) { + previous.releaseMysqlTransport(); + } BackendProxyHandler replacement = new BackendProxyHandler(this); try { replacement.load(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 82d02e2eb8..577e11afa3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -276,6 +276,14 @@ public synchronized void releaseMqttTransport() { } } + /** Stops a MySQL messenger before another subscriber is activated for the same method. */ + public synchronized void releaseMysqlTransport() { + if (backendMysqlMessenger != null) { + backendMysqlMessenger.shutdown(); + backendMysqlMessenger = null; + } + } + /** * Loads and initializes the bungee handler with the configured method. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 9dc5100130..69f4fb8565 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -76,6 +76,7 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); move(backupStaging, backup); + if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); move(staging, target); installed = true; reload.run(fileName); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 85a132188a..83294e0609 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -10,6 +10,7 @@ import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; import com.bencodez.simpleapi.servercomm.mqtt.MqttHandler; +import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; class BackendProxyHandlerLifecycleTest { @@ -55,4 +56,18 @@ void releasesMqttSubscriberBeforeSameMethodReplacement() throws Exception { verify(mqtt).disconnect(); assertNull(handler.getMqttHandler()); } + + @Test + void releasesMysqlSubscriberBeforeSameMethodReplacement() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + MySqlMessenger messenger = mock(MySqlMessenger.class); + Field field = BackendProxyHandler.class.getDeclaredField("backendMysqlMessenger"); + field.setAccessible(true); + field.set(handler, messenger); + + handler.releaseMysqlTransport(); + + verify(messenger).shutdown(); + assertNull(handler.getBackendMysqlMessenger()); + } } From 45aa84d2093cd1c5a362ea4cd73311a1b02c8ff3 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 25 Aug 2026 21:40:58 -0600 Subject: [PATCH 30/73] Stop replaced global data timers --- .../backendproxy/BackendProxyHandler.java | 9 +++++++++ .../BackendProxyHandlerLifecycleTest.java | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 577e11afa3..9f3d2557d7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -209,6 +209,7 @@ public boolean checkGlobalDataTimeValue(DataValue data) { */ public void close() { plugin.deactivateBackendPluginMessageHandler(globalMessageHandler); + releaseGlobalDataTimer(); if (presenceManager != null) { presenceManager.stop(); @@ -244,6 +245,14 @@ public void close() { } } + /** Stops periodic global-data work owned by this handler before it is discarded. */ + public synchronized void releaseGlobalDataTimer() { + if (timer != null) { + timer.shutdownNow(); + timer = null; + } + } + /** Releases the listening port before a same-address socket replacement is created. */ public synchronized void releaseSocketListener() { if (socketHandler != null) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 83294e0609..bf392bb81a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -5,6 +5,7 @@ import static org.mockito.Mockito.verify; import java.lang.reflect.Field; +import java.util.concurrent.ScheduledExecutorService; import org.junit.jupiter.api.Test; @@ -70,4 +71,18 @@ void releasesMysqlSubscriberBeforeSameMethodReplacement() throws Exception { verify(messenger).shutdown(); assertNull(handler.getBackendMysqlMessenger()); } + + @Test + void stopsGlobalDataTimerWhenHandlerIsReplaced() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + Field field = BackendProxyHandler.class.getDeclaredField("timer"); + field.setAccessible(true); + field.set(handler, timer); + + handler.releaseGlobalDataTimer(); + + verify(timer).shutdownNow(); + assertNull(handler.getTimer()); + } } From e189ba9894371a447cd0a25394548c6140592153 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 17:26:27 -0600 Subject: [PATCH 31/73] Retain hosted process after failed shutdown --- .../votingplugin/proxy/VotingPluginProxy.java | 2 +- .../proxy/control/HostedControlManager.java | 35 ++++++++++++++---- .../proxy/VotingPluginProxyLifecycleTest.java | 19 ++++++++++ .../control/HostedControlManagerTest.java | 36 +++++++++++++++++++ 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 1d0fb5344d..ecd2f1a164 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1612,13 +1612,13 @@ private void stopControlServicesLocked(boolean waitForHosted) { connector.close(); } HostedControlManager manager = hostedControlManager; - hostedControlManager = null; if (manager != null) { if (waitForHosted) { manager.closeAndWait(); } else { manager.close(); } + if (hostedControlManager == manager) hostedControlManager = null; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 1998e4ccc0..7e09f85163 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -304,27 +304,50 @@ private void close(boolean waitForProcess) { closed = true; status = Status.STOPPED; process = managedProcess; - managedProcess = null; } Future task = activeTask; if (task != null) { task.cancel(true); } - if (process != null) { - stopProcess(process, waitForProcess); + RuntimeException shutdownFailure = null; + try { + if (process != null) { + stopProcess(process, waitForProcess); + if (waitForProcess) { + clearManagedProcess(process); + } else { + process.onExit().whenComplete((ignored, failure) -> clearManagedProcess(process)); + } + } + } catch (RuntimeException failure) { + shutdownFailure = failure; + } finally { + executor.shutdownNow(); } - executor.shutdownNow(); if (waitForProcess) { long waitSeconds = (long) settings.downloadTimeoutSeconds() + settings.startupTimeoutSeconds() + 5L; try { if (!executor.awaitTermination(waitSeconds, TimeUnit.SECONDS)) { - throw new IllegalStateException("Hosted Control worker did not stop before reload"); + IllegalStateException workerFailure = new IllegalStateException( + "Hosted Control worker did not stop before reload"); + if (shutdownFailure != null) workerFailure.addSuppressed(shutdownFailure); + throw workerFailure; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); - throw new IllegalStateException("Interrupted while waiting for the hosted Control worker", e); + IllegalStateException interrupted = new IllegalStateException( + "Interrupted while waiting for the hosted Control worker", e); + if (shutdownFailure != null) interrupted.addSuppressed(shutdownFailure); + throw interrupted; } } + if (shutdownFailure != null) throw shutdownFailure; + } + + private void clearManagedProcess(ManagedProcess process) { + synchronized (processLifecycle) { + if (managedProcess == process) managedProcess = null; + } } static void stopProcess(ManagedProcess process, boolean wait) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index 227ceb281e..16278ca77f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -3,6 +3,8 @@ import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.lang.reflect.Field; import java.lang.reflect.Method; @@ -18,9 +20,26 @@ import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; import com.bencodez.votingplugin.tests.VotingPluginProxyTestImpl; +import com.bencodez.votingplugin.proxy.control.HostedControlManager; class VotingPluginProxyLifecycleTest { + @Test + void retainsHostedManagerWhenBoundedShutdownFails() throws Exception { + VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + HostedControlManager manager = mock(HostedControlManager.class); + doThrow(new IllegalStateException("still running")).when(manager).closeAndWait(); + Field hosted = VotingPluginProxy.class.getDeclaredField("hostedControlManager"); + hosted.setAccessible(true); + hosted.set(proxy, manager); + Method stop = VotingPluginProxy.class.getDeclaredMethod("stopControlServices", boolean.class); + stop.setAccessible(true); + + assertThrows(java.lang.reflect.InvocationTargetException.class, () -> stop.invoke(proxy, true)); + + assertSame(manager, hosted.get(proxy)); + } + @Test void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { ClientHandler failing = mock(ClientHandler.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 7934ba893f..86b2d1114c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -256,6 +257,26 @@ void forcedProcessTerminationIsAwaitedBeforeReturning() { assertEquals(2, process.waits.get()); } + @Test + void failedForcedTerminationRetainsManagedProcessForRetry() throws Exception { + Path root = directory.toAbsolutePath().normalize(); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager manager = new HostedControlManager( + settings(root.resolve("control.jar"), root.resolve("data"), false, false, digest("x".getBytes()), 1), + executor, (source, target, maximum, timeout) -> { }, new FakeLauncher(), + (endpoint, timeout) -> true, millis -> { }, System::nanoTime, message -> { }); + UnkillableProcess process = new UnkillableProcess(); + java.lang.reflect.Field managed = HostedControlManager.class.getDeclaredField("managedProcess"); + managed.setAccessible(true); + managed.set(manager, process); + + assertThrows(IllegalStateException.class, manager::closeAndWait); + + assertSame(process, managed.get(manager)); + assertTrue(process.forciblyDestroyed); + assertEquals(2, process.waits.get()); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); @@ -315,4 +336,19 @@ private static final class StubbornProcess implements HostedControlManager.Manag } @Override public CompletableFuture onExit() { return exit; } } + + private static final class UnkillableProcess implements HostedControlManager.ManagedProcess { + private final AtomicInteger waits = new AtomicInteger(); + private final CompletableFuture exit = new CompletableFuture<>(); + private volatile boolean forciblyDestroyed; + + @Override public boolean isAlive() { return true; } + @Override public void destroy() { } + @Override public void destroyForcibly() { forciblyDestroyed = true; } + @Override public boolean waitFor(long timeout, TimeUnit unit) { + waits.incrementAndGet(); + return false; + } + @Override public CompletableFuture onExit() { return exit; } + } } From 793bdf9a27cf50fef731ff47cc09b3145d6cfd0c Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 17:31:42 -0600 Subject: [PATCH 32/73] Preserve plugin-message relay during replacement --- .../backendproxy/BackendProxyHandler.java | 4 +++- .../BackendProxyHandlerLifecycleTest.java | 23 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 263b3cc0cb..a961ef439f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -98,7 +98,9 @@ public void close() { /** Releases a same-method subscriber/listener before its replacement starts. */ public void prepareForReplacement(BungeeMethod replacementMethod) { - if (method == replacementMethod) transportManager.prepareForReplacement(); + if (method == replacementMethod && method != BungeeMethod.PLUGINMESSAGING) { + transportManager.prepareForReplacement(); + } } /** Fails a configuration apply when its selected transport did not initialize. */ diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index a798eccf66..e77dd46d88 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import java.lang.reflect.Field; import java.util.concurrent.ScheduledExecutorService; @@ -16,11 +17,33 @@ import com.bencodez.votingplugin.backendproxy.global.BackendGlobalDataSync; import com.bencodez.votingplugin.backendproxy.transport.MqttBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.MysqlBackendProxyTransport; +import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransport; +import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransportManager; import com.bencodez.votingplugin.backendproxy.transport.RedisBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.SocketBackendProxyTransport; +import com.bencodez.votingplugin.proxy.BungeeMethod; class BackendProxyHandlerLifecycleTest { + @Test + void keepsPluginMessageRelayActiveUntilAtomicTargetSwap() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + Field method = BackendProxyHandler.class.getDeclaredField("method"); + method.setAccessible(true); + method.set(handler, BungeeMethod.PLUGINMESSAGING); + Field managerField = BackendProxyHandler.class.getDeclaredField("transportManager"); + managerField.setAccessible(true); + BackendProxyTransportManager manager = (BackendProxyTransportManager) managerField.get(handler); + BackendProxyTransport transport = mock(BackendProxyTransport.class); + Field transportField = BackendProxyTransportManager.class.getDeclaredField("transport"); + transportField.setAccessible(true); + transportField.set(manager, transport); + + handler.prepareForReplacement(BungeeMethod.PLUGINMESSAGING); + + verifyNoInteractions(transport); + } + @Test void releasesSocketListenerBeforeSamePortReplacement() throws Exception { SocketBackendProxyTransport handler = new SocketBackendProxyTransport(null); From f4567a8bb66067897b7d8c47a09afe63f526e2fa Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 17:40:33 -0600 Subject: [PATCH 33/73] Preserve state and rollback safety across reloads --- .../com/bencodez/votingplugin/VotingPluginMain.java | 6 ++++-- .../backendproxy/BackendProxyHandler.java | 7 ++++++- .../PluginMessagingBackendProxyTransport.java | 6 +++--- .../control/BackendConfigurationService.java | 7 +++++-- .../votingplugin/proxy/bungee/BungeeConfig.java | 11 +++++++++-- .../votingplugin/proxy/velocity/VelocityConfig.java | 11 +++++++++-- .../BackendProxyHandlerLifecycleTest.java | 11 +++++++++++ 7 files changed, 47 insertions(+), 12 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 1803d0124a..11b433329f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -57,6 +57,7 @@ import com.bencodez.simpleapi.updater.Updater; import com.bencodez.votingplugin.broadcast.BroadcastHandler; import com.bencodez.votingplugin.backendproxy.BackendProxyHandler; +import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; import com.bencodez.votingplugin.backgroundtask.VotingPluginBackgroundTask; import com.bencodez.votingplugin.backendproxy.BackendProxyRewardRegistrar; import com.bencodez.votingplugin.broadcast.BroadcastSettings; @@ -155,6 +156,7 @@ public class VotingPluginMain extends AdvancedCorePlugin { @Getter private BackendProxyHandler backendProxyHandler; + private final ProcessedVoteCache backendProcessedVoteCache = new ProcessedVoteCache(); private final AtomicReference backendPluginMessageTarget = new AtomicReference<>(); private PluginMessageHandler backendPluginMessageRelay; private BackendControlConnector backendControlConnector; @@ -440,7 +442,7 @@ public boolean isVoteSite(String voteSite) { } private void loadBungeeHandler() { - backendProxyHandler = new BackendProxyHandler(this); + backendProxyHandler = new BackendProxyHandler(this, backendProcessedVoteCache); backendProxyHandler.load(); if (getOptions().getServer().equalsIgnoreCase("PleaseSet")) { @@ -757,7 +759,7 @@ public synchronized void restartBackendProxyHandler() { } BungeeMethod replacementMethod = BungeeMethod.getByName(bungeeSettings.getBungeeMethod()); if (previous != null) previous.prepareForReplacement(replacementMethod); - BackendProxyHandler replacement = new BackendProxyHandler(this); + BackendProxyHandler replacement = new BackendProxyHandler(this, backendProcessedVoteCache); try { replacement.load(); replacement.validateTransport(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index a961ef439f..3a18d1a76c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -34,7 +34,7 @@ public class BackendProxyHandler implements Listener { private final VotingPluginMain plugin; - private final ProcessedVoteCache processedVoteCache = new ProcessedVoteCache(); + private final ProcessedVoteCache processedVoteCache; private final BackendProxyTransportManager transportManager; private final BackendGlobalDataSync globalDataSync; @@ -48,7 +48,12 @@ public class BackendProxyHandler implements Listener { private GlobalMessageHandler globalMessageHandler; public BackendProxyHandler(VotingPluginMain plugin) { + this(plugin, new ProcessedVoteCache()); + } + + public BackendProxyHandler(VotingPluginMain plugin, ProcessedVoteCache processedVoteCache) { this.plugin = plugin; + this.processedVoteCache = java.util.Objects.requireNonNull(processedVoteCache, "processedVoteCache"); transportManager = new BackendProxyTransportManager(plugin); globalDataSync = new BackendGlobalDataSync(plugin, this::sendEnvelope); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java index 977770da03..d6a24ff5cb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/PluginMessagingBackendProxyTransport.java @@ -20,12 +20,12 @@ public PluginMessagingBackendProxyTransport(VotingPluginMain plugin) { public void start(GlobalMessageHandler messageHandler) { this.messageHandler = messageHandler; plugin.registerBungeeChannels(plugin.getBungeeSettings().getPluginMessagingChannel()); - plugin.getPluginMessaging().setEncryptionHandler(null); + EncryptionHandler encryptionHandler = null; if (plugin.getBungeeSettings().isPluginMessageEncryption()) { - EncryptionHandler encryptionHandler = new EncryptionHandler(plugin.getName(), + encryptionHandler = new EncryptionHandler(plugin.getName(), new File(plugin.getDataFolder(), "secretkey.key")); - plugin.getPluginMessaging().setEncryptionHandler(encryptionHandler); } + plugin.getPluginMessaging().setEncryptionHandler(encryptionHandler); plugin.getPluginMessaging().setDebug(plugin.getBungeeSettings().isBungeeDebug()); plugin.activateBackendPluginMessageHandler(messageHandler); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 69f4fb8565..54c6e33384 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -93,7 +93,7 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte if (!revision(readRaw(target, false)).equals(installedRevision)) { throw new IOException("Managed configuration changed while reload failed; backup was not restored"); } - copyBackupNoFollow(backup, target); + copyBackupNoFollow(backup, target, installedRevision); reload.run(fileName); rolledBack = true; } catch (Exception rollbackFailure) { @@ -111,7 +111,7 @@ private static void rejectSymbolicBackup(Path backup) throws IOException { if (Files.isSymbolicLink(backup)) throw new IOException("Symbolic configuration backups are not allowed"); } - private static void copyBackupNoFollow(Path backup, Path target) throws IOException { + private static void copyBackupNoFollow(Path backup, Path target, String expectedTargetRevision) throws IOException { if (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS)) { throw new IOException("Control backup is unavailable or unsafe"); } @@ -121,6 +121,9 @@ private static void copyBackupNoFollow(Path backup, Path target) throws IOExcept Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { Files.copy(Channels.newInputStream(source), staging, StandardCopyOption.REPLACE_EXISTING); } + if (!revision(readRaw(target, false)).equals(expectedTargetRevision)) { + throw new IOException("Managed configuration changed while rollback was staged"); + } move(staging, target); } finally { Files.deleteIfExists(staging); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index 6bad31a653..62673dfa3a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -568,8 +568,15 @@ public synchronized void rollbackControlProxyRouting() throws IOException { throw new StaleControlRevisionException(); } Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); - Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); - atomicReplace(stage, target); + try { + Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); + if (!java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + atomicReplace(stage, target); + } finally { + Files.deleteIfExists(stage); + } controlInstalledSnapshot = null; data = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 73f1bd6907..3aec9593f7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -91,8 +91,15 @@ public synchronized void rollbackControlProxyRouting() throws IOException { throw new StaleControlRevisionException(); } Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); - Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); - atomicReplace(stage, target); + try { + Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); + if (!java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + atomicReplace(stage, target); + } finally { + Files.deleteIfExists(stage); + } controlInstalledSnapshot = null; loadControlConfiguration(); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index e77dd46d88..9e9c2d0dcc 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.backendproxy; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verifyNoInteractions; @@ -15,6 +16,7 @@ import com.bencodez.simpleapi.servercomm.mysql.MySqlMessenger; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; import com.bencodez.votingplugin.backendproxy.global.BackendGlobalDataSync; +import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; import com.bencodez.votingplugin.backendproxy.transport.MqttBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.MysqlBackendProxyTransport; import com.bencodez.votingplugin.backendproxy.transport.BackendProxyTransport; @@ -25,6 +27,15 @@ class BackendProxyHandlerLifecycleTest { + @Test + void sharesVoteDeduplicationAcrossHandlerReplacement() { + ProcessedVoteCache cache = new ProcessedVoteCache(); + BackendProxyHandler previous = new BackendProxyHandler(null, cache); + BackendProxyHandler replacement = new BackendProxyHandler(null, cache); + + assertSame(previous.getProcessedWireVotes(), replacement.getProcessedWireVotes()); + } + @Test void keepsPluginMessageRelayActiveUntilAtomicTargetSwap() throws Exception { BackendProxyHandler handler = new BackendProxyHandler(null); From 7ef07187e62b2db6d560bf8fda0fa9346d34b925 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 17:51:56 -0600 Subject: [PATCH 34/73] Await Control operations during shutdown --- .../votingplugin/proxy/VotingPluginProxy.java | 2 +- .../proxy/control/ControlConnector.java | 43 +++++++++++++---- .../proxy/VotingPluginProxyLifecycleTest.java | 17 +++++++ .../proxy/control/ControlConnectorTest.java | 46 ++++++++++++++++++- 4 files changed, 97 insertions(+), 11 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index ecd2f1a164..b12a4f0685 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1607,9 +1607,9 @@ private void startControlServicesLocked() { private void stopControlServicesLocked(boolean waitForHosted) { ControlConnector connector = controlConnector; - controlConnector = null; if (connector != null) { connector.close(); + if (controlConnector == connector) controlConnector = null; } HostedControlManager manager = hostedControlManager; if (manager != null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 4a25655232..eba148e17b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -52,6 +52,7 @@ public final class ControlConnector implements AutoCloseable { private static final Set BASE_CAPABILITIES = Set.of("presence.snapshot"); private static final String CONFIGURATION_CAPABILITY = "config.proxy-routing.v1"; private static final long MAX_BACKOFF_MILLIS = TimeUnit.MINUTES.toMillis(5); + private static final long OPERATION_SHUTDOWN_TIMEOUT_MILLIS = TimeUnit.SECONDS.toMillis(65); private final Settings settings; private final ScheduledExecutorService scheduler; @@ -70,6 +71,7 @@ public final class ControlConnector implements AutoCloseable { private volatile long snapshotSequence; private volatile ScheduledFuture scheduled; private volatile CompletableFuture activeRequest; + private volatile CompletableFuture activeOperation; private volatile Status status = Status.STARTING; public ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, @@ -174,6 +176,8 @@ void cycle() { return; } activeRequest = primary; + CompletableFuture operationDone = new CompletableFuture<>(); + activeOperation = operationDone; CompletableFuture operation = primary.thenCompose(response -> { handlePrimaryResponse(response, !registered); registered = true; @@ -188,15 +192,23 @@ void cycle() { }); }); operation.whenComplete((ignored, failure) -> { - activeRequest = null; - inFlight.set(false); - if (closed) { - return; - } - if (failure == null) { - onSuccess(); - } else { - onFailure(unwrap(failure)); + try { + activeRequest = null; + inFlight.set(false); + if (!closed) { + if (failure == null) { + onSuccess(); + } else { + onFailure(unwrap(failure)); + } + } + } finally { + if (failure == null) { + operationDone.complete(null); + } else { + operationDone.completeExceptionally(failure); + } + if (activeOperation == operationDone) activeOperation = null; } }); } @@ -470,6 +482,19 @@ public void close() { if (request != null) { request.cancel(true); } + CompletableFuture operation = activeOperation; + if (operation != null) { + try { + operation.get(OPERATION_SHUTDOWN_TIMEOUT_MILLIS, TimeUnit.MILLISECONDS); + } catch (java.util.concurrent.CancellationException | java.util.concurrent.ExecutionException ignored) { + // Cancellation/failure completes the chain and makes transport shutdown safe. + } catch (java.util.concurrent.TimeoutException e) { + throw new IllegalStateException("Control operation did not stop before connector shutdown", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for the Control operation", e); + } + } transport.close(); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index 16278ca77f..df745ba910 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -21,9 +21,26 @@ import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; import com.bencodez.votingplugin.tests.VotingPluginProxyTestImpl; import com.bencodez.votingplugin.proxy.control.HostedControlManager; +import com.bencodez.votingplugin.proxy.control.ControlConnector; class VotingPluginProxyLifecycleTest { + @Test + void retainsConnectorWhenOperationShutdownFails() throws Exception { + VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + ControlConnector connector = mock(ControlConnector.class); + doThrow(new IllegalStateException("operation still running")).when(connector).close(); + Field control = VotingPluginProxy.class.getDeclaredField("controlConnector"); + control.setAccessible(true); + control.set(proxy, connector); + Method stop = VotingPluginProxy.class.getDeclaredMethod("stopControlServices", boolean.class); + stop.setAccessible(true); + + assertThrows(java.lang.reflect.InvocationTargetException.class, () -> stop.invoke(proxy, true)); + + assertSame(connector, control.get(proxy)); + } + @Test void retainsHostedManagerWhenBoundedShutdownFails() throws Exception { VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index ef74e55980..152bbf7cf3 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -13,8 +13,10 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -155,6 +157,46 @@ class ControlConnectorTest { assertEquals(Status.CONNECTED, connector.status()); } + @Test void closeWaitsForClaimedApplyToFinish() throws Exception { + connector.close(); + CountDownLatch persistEntered = new CountDownLatch(1); + CountDownLatch releasePersist = new CountDownLatch(1); + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + ProxyRoutingConfigurationService service = new ProxyRoutingConfigurationService(new NoOpPlatform() { + @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) + throws java.io.IOException { + persistEntered.countDown(); + try { + releasePersist.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new java.io.IOException(e); + } + } + }); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L, service); + transport.acceptConfiguration = true; + transport.operationClaim = new CompletableFuture<>(); + connector.cycle(); + try { + CompletableFuture executing = CompletableFuture.runAsync(() -> transport.operationClaim.complete( + new Response(200, "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"type\":\"APPLY\",\"expectedRevision\":\"" + current.revision() + "\"," + + "\"configuration\":{\"sendVotesToAllServers\":true,\"blockedServers\":[]}}"))); + assertTrue(persistEntered.await(2, TimeUnit.SECONDS)); + CompletableFuture closing = CompletableFuture.runAsync(connector::close); + assertThrows(java.util.concurrent.TimeoutException.class, () -> closing.get(100, TimeUnit.MILLISECONDS)); + + releasePersist.countDown(); + executing.get(2, TimeUnit.SECONDS); + closing.get(2, TimeUnit.SECONDS); + assertEquals(Status.STOPPED, connector.status()); + } finally { + releasePersist.countDown(); + } + } + @Test void backoffIsBoundedExponentialAndJitteredWithoutSleeping() { assertEquals(1000, ControlConnector.backoffMillis(1, 0)); assertEquals(2000, ControlConnector.backoffMillis(2, 0)); @@ -191,6 +233,7 @@ private static final class FakeTransport implements Transport { private CompletableFuture stalled; private RuntimeException synchronousFailure; private boolean acceptConfiguration; + private CompletableFuture operationClaim; @Override public CompletableFuture send(Request request) { @@ -206,6 +249,7 @@ public CompletableFuture send(Request request) { return result; } if (request.path().endsWith("/operations")) { + if (operationClaim != null) return operationClaim; return CompletableFuture.completedFuture(new Response(204, "")); } if (!request.path().endsWith("/presence")) { @@ -227,7 +271,7 @@ public CompletableFuture send(Request request) { } } - private static final class NoOpPlatform implements ProxyRoutingConfigurationService.Platform { + private static class NoOpPlatform implements ProxyRoutingConfigurationService.Platform { @Override public ProxyRoutingConfiguration read() { return new ProxyRoutingConfiguration(false, List.of()); } @Override public java.util.Set configuredServers() { return java.util.Set.of(); } @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) { } From 00ee58b9146bd2e67393602aae12944989a7a2bb Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 18:00:19 -0600 Subject: [PATCH 35/73] Defer hosted retries until retained process exits --- .../proxy/control/HostedControlManager.java | 42 ++++++++++++++++- .../control/HostedControlManagerTest.java | 45 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 7e09f85163..0e0f935201 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -60,6 +60,7 @@ public final class HostedControlManager implements AutoCloseable { private volatile boolean closed; private volatile Status status = Status.STARTING; private volatile ManagedProcess managedProcess; + private volatile ManagedProcess exitAwaitedProcess; private volatile Future activeTask; private volatile int failures; private volatile String quarantinedSha256; @@ -111,6 +112,11 @@ void runOnce() { return; } try { + ManagedProcess retained = managedProcess; + if (retained != null) { + if (retained.isAlive()) return; + clearManagedProcess(retained); + } boolean updated = prepareArtifact(); if (closed) { return; @@ -217,6 +223,13 @@ private ManagedProcess launch(Path artifact) throws IOException { status = Status.STARTING_PROCESS; synchronized (processLifecycle) { if (closed) throw new IOException("Hosted Control manager is closed"); + if (managedProcess != null) { + if (managedProcess.isAlive()) { + throw new IOException("The previous hosted Control process is still running"); + } + managedProcess = null; + exitAwaitedProcess = null; + } ManagedProcess process = launcher.launch(settings, artifact); managedProcess = process; return process; @@ -267,10 +280,34 @@ private void monitor(ManagedProcess process) { private void fail() { status = Status.FAILED; + ManagedProcess process = managedProcess; + if (process != null && process.isAlive()) { + failWhileProcessIsAlive(process); + return; + } + if (process != null) clearManagedProcess(process); logger.accept("Control could not be provisioned or started; VotingPlugin remains unaffected and will retry"); scheduleRetry(); } + private void failWhileProcessIsAlive(ManagedProcess process) { + status = Status.FAILED; + synchronized (processLifecycle) { + if (closed || managedProcess != process || exitAwaitedProcess == process) return; + exitAwaitedProcess = process; + } + logger.accept("Control could not be stopped; provisioning will retry only after that process exits"); + process.onExit().whenComplete((ignored, failure) -> { + boolean retry; + synchronized (processLifecycle) { + if (exitAwaitedProcess == process) exitAwaitedProcess = null; + retry = !closed && managedProcess == process; + if (retry) managedProcess = null; + } + if (retry) scheduleRetry(); + }); + } + private void scheduleRetry() { if (closed) { return; @@ -346,7 +383,10 @@ private void close(boolean waitForProcess) { private void clearManagedProcess(ManagedProcess process) { synchronized (processLifecycle) { - if (managedProcess == process) managedProcess = null; + if (managedProcess == process) { + managedProcess = null; + if (exitAwaitedProcess == process) exitAwaitedProcess = null; + } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 86b2d1114c..f1d2926191 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -19,6 +19,7 @@ import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -277,6 +278,35 @@ void failedForcedTerminationRetainsManagedProcessForRetry() throws Exception { assertEquals(2, process.waits.get()); } + @Test + void failedHealthTerminationDefersRetryUntilRetainedProcessExits() throws Exception { + byte[] release = "manual-release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control.jar"); + Files.write(jar, release); + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); + RetainedProcess process = new RetainedProcess(); + AtomicInteger launches = new AtomicInteger(); + AtomicLong clock = new AtomicLong(); + HostedControlManager manager = new HostedControlManager( + settings(jar, root.resolve("data"), false, false, digest(release), 1), executor, + (source, target, maximum, timeout) -> { }, (settings, artifact) -> { + launches.incrementAndGet(); + return process; + }, (endpoint, timeout) -> false, + millis -> clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(millis)), clock::get, message -> { }); + + manager.runOnce(); + manager.runOnce(); + + assertEquals(HostedControlManager.Status.FAILED, manager.status()); + assertEquals(1, launches.get()); + assertTrue(executor.getQueue().isEmpty()); + process.exit(); + assertEquals(1, executor.getQueue().size()); + manager.close(); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); @@ -351,4 +381,19 @@ private static final class UnkillableProcess implements HostedControlManager.Man } @Override public CompletableFuture onExit() { return exit; } } + + private static final class RetainedProcess implements HostedControlManager.ManagedProcess { + private final CompletableFuture exit = new CompletableFuture<>(); + private volatile boolean alive = true; + + @Override public boolean isAlive() { return alive; } + @Override public void destroy() { } + @Override public void destroyForcibly() { } + @Override public boolean waitFor(long timeout, TimeUnit unit) { return false; } + @Override public CompletableFuture onExit() { return exit; } + private void exit() { + alive = false; + exit.complete(null); + } + } } From 0ab24c0abb14d479bee1a92164807e0d1f778c3d Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 18:01:45 -0600 Subject: [PATCH 36/73] Fix Control shutdown test signature --- .../votingplugin/proxy/control/ControlConnectorTest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 152bbf7cf3..518d9d4c83 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -274,7 +274,8 @@ public CompletableFuture send(Request request) { private static class NoOpPlatform implements ProxyRoutingConfigurationService.Platform { @Override public ProxyRoutingConfiguration read() { return new ProxyRoutingConfiguration(false, List.of()); } @Override public java.util.Set configuredServers() { return java.util.Set.of(); } - @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) { } + @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) + throws java.io.IOException { } @Override public void rollback() { } @Override public void reload() { } } From 52221d01a280940dbbcbc6cd9a3201e472dc3ef9 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 18:03:40 -0600 Subject: [PATCH 37/73] Model hosted restart after child exit --- .../votingplugin/proxy/control/HostedControlManagerTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index f1d2926191..c09e64e3ff 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -164,6 +164,7 @@ void failedUpdateRollsBackAndStartsPreviousRelease() throws Exception { assertEquals("previous-release", Files.readString(jar)); assertEquals("bad-new-release", Files.readString(settings.failedFile())); assertTrue(launcher.processes.get(0).destroyed); + launcher.processes.get(1).destroy(); Files.writeString(jar, "corrupted-after-rollback"); manager.runOnce(); assertEquals(HostedControlManager.Status.FAILED, manager.status()); From d660292d16a564af475d583b26d69c9c265c7bc5 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 18:12:38 -0600 Subject: [PATCH 38/73] Validate backend setup and require atomic activation --- .../control/BackendConfigurationService.java | 17 +++++++++++------ .../BackendConfigurationServiceTest.java | 12 ++++++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 54c6e33384..f6c48d28b7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -4,7 +4,6 @@ import java.nio.charset.CharacterCodingException; import java.nio.charset.CodingErrorAction; import java.nio.charset.StandardCharsets; -import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; @@ -27,6 +26,8 @@ import org.bukkit.configuration.InvalidConfigurationException; import org.bukkit.configuration.file.YamlConfiguration; +import com.bencodez.votingplugin.proxy.BungeeMethod; + /** Safe, revisioned access to every user-facing VotingPlugin YAML configuration file. */ public final class BackendConfigurationService { public static final String REDACTED = "__VOTINGPLUGIN_CONTROL_REDACTED__"; @@ -170,7 +171,7 @@ private QuickProposal quickProposal(String preset, Map options, if (proxy) { String server = option(options, "server", "[A-Za-z0-9][A-Za-z0-9._-]{0,63}"); yaml.set("Server", server); - yaml.set("BungeeMethod", options.getOrDefault("method", "PLUGINMESSAGING")); + yaml.set("BungeeMethod", bungeeMethodOption(options)); } return new QuickProposal(fileName, yaml.saveToString()); } @@ -336,11 +337,15 @@ private static String revision(String content) { } private static void move(Path source, Path target) throws IOException { - try { - Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (AtomicMoveNotSupportedException e) { - Files.move(source, target, StandardCopyOption.REPLACE_EXISTING); + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } + + private static String bungeeMethodOption(Map options) { + String value = options == null ? "PLUGINMESSAGING" : options.getOrDefault("method", "PLUGINMESSAGING"); + for (BungeeMethod method : BungeeMethod.values()) { + if (method.name().equalsIgnoreCase(value)) return method.name(); } + throw new IllegalArgumentException("quick setup option method is invalid"); } private static String option(Map options, String name, String pattern) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 86ee2408ca..111a1f8a5b 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -158,4 +158,16 @@ class BackendConfigurationServiceTest { "giveAllPlayers", "false", "onlineOnly", "true")); assertTrue(party.proposal().content().contains("VotesRequired: 25")); } + + @Test void proxyBackendQuickSetupRejectsUnknownTransportMethods() throws Exception { + Files.writeString(directory.resolve("BungeeSettings.yml"), + "UseBungeecord: false\nServer: PleaseSet\nBungeeMethod: PLUGINMESSAGING\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + + assertThrows(IllegalArgumentException.class, () -> service.previewQuickSetup("proxy-backend", + Map.of("server", "lobby", "method", "NOT_A_TRANSPORT"))); + BackendConfigurationService.QuickPreview mqtt = service.previewQuickSetup("proxy-backend", + Map.of("server", "lobby", "method", "mqtt")); + assertTrue(mqtt.proposal().content().contains("BungeeMethod: MQTT")); + } } From e43c30e9cf7f70d683001600f168b45227762634 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 18:24:37 -0600 Subject: [PATCH 39/73] Drain Control operations before connector shutdown --- .../votingplugin/VotingPluginMain.java | 14 +-- .../control/BackendControlConnector.java | 87 +++++++++++++++++-- .../proxy/control/ControlConnector.java | 9 +- .../BackendControlConnectorProtocolTest.java | 16 ++++ .../proxy/control/ControlConnectorTest.java | 32 +++++++ 5 files changed, 143 insertions(+), 15 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 11b433329f..6808fc6808 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -744,8 +744,11 @@ private void startBackendControlConnector() { /** Refreshes the optional Control connector after its Config.yml settings were applied. */ public synchronized void restartBackendControlConnector() { - if (backendControlConnector != null) backendControlConnector.close(); - backendControlConnector = null; + BackendControlConnector connector = backendControlConnector; + if (connector != null) { + connector.close(); + if (backendControlConnector == connector) backendControlConnector = null; + } startBackendControlConnector(); } @@ -881,9 +884,10 @@ public void onFinish() { @Override public void onUnLoad() { - if (backendControlConnector != null) { - backendControlConnector.close(); - backendControlConnector = null; + BackendControlConnector connector = backendControlConnector; + if (connector != null) { + connector.close(); + if (backendControlConnector == connector) backendControlConnector = null; } if (getBackendProxyHandler() != null) { try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 6e3a646fb3..563f42fa9a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -14,7 +14,9 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; +import java.util.concurrent.Future; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.ThreadFactory; @@ -22,6 +24,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.regex.Pattern; +import org.bukkit.Bukkit; import org.bukkit.configuration.ConfigurationSection; import com.bencodez.votingplugin.VotingPluginMain; @@ -36,6 +39,7 @@ public final class BackendControlConnector implements AutoCloseable { private static final int PROTOCOL_VERSION = 1; private static final int MAX_RESPONSE_BYTES = 768 * 1024; + private static final long SHUTDOWN_TIMEOUT_SECONDS = 65; private static final Pattern NODE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,63}"); private static final Set CAPABILITIES = Set.of("config.files.v1", "config.quick-setup.v1"); @@ -48,12 +52,15 @@ public final class BackendControlConnector implements AutoCloseable { private final UUID sessionId = UUID.randomUUID(); private final Map completed = new LinkedHashMap<>(); private final AtomicBoolean running = new AtomicBoolean(); + private final Object operationLifecycle = new Object(); private volatile boolean closed; private volatile boolean registered; private volatile boolean operationsAccepted; private volatile boolean quickSetupsAccepted; private volatile int failures; private volatile ScheduledFuture scheduled; + private volatile Future activeReload; + private volatile CompletableFuture activeOperation; private BackendControlConnector(VotingPluginMain plugin, Settings settings, String credential) { this.plugin = plugin; @@ -67,12 +74,27 @@ private BackendControlConnector(VotingPluginMain plugin, Settings settings, Stri executor = Executors.newSingleThreadScheduledExecutor(factory); http = HttpClient.newBuilder().connectTimeout(Duration.ofMillis(settings.connectTimeoutMillis())) .followRedirects(HttpClient.Redirect.NEVER).build(); - configurations = new BackendConfigurationService(plugin.getDataFolder().toPath(), fileName -> - plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { - plugin.reload(); - if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(); - return null; - }).get(30, TimeUnit.SECONDS)); + configurations = new BackendConfigurationService(plugin.getDataFolder().toPath(), this::reloadConfiguration); + } + + private void reloadConfiguration(String fileName) throws Exception { + Future reload; + synchronized (operationLifecycle) { + if (closed) throw new IllegalStateException("Bukkit Control connector is stopping"); + reload = plugin.getServer().getScheduler().callSyncMethod(plugin, () -> { + plugin.reload(); + if ("BungeeSettings.yml".equals(fileName)) plugin.restartBackendProxyHandler(); + return null; + }); + activeReload = reload; + } + try { + reload.get(30, TimeUnit.SECONDS); + } finally { + synchronized (operationLifecycle) { + if (activeReload == reload) activeReload = null; + } + } } public static BackendControlConnector create(VotingPluginMain plugin) throws IOException { @@ -123,7 +145,22 @@ private void cycle() { throw incompatible; } registered = true; - if (operationsAccepted) claimAndExecute(); + if (closed) return; + if (operationsAccepted) { + CompletableFuture operation = new CompletableFuture<>(); + synchronized (operationLifecycle) { + if (closed) return; + activeOperation = operation; + } + try { + claimAndExecute(); + } finally { + operation.complete(null); + synchronized (operationLifecycle) { + if (activeOperation == operation) activeOperation = null; + } + } + } if (failures > 0) plugin.getLogger().info("[Control] Bukkit configuration connector recovered"); failures = 0; } catch (Exception failure) { @@ -174,6 +211,7 @@ private void claimAndExecute() throws Exception { Response response = send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations", body); if (response.status() == 204) return; JsonObject task = requireObject(response, 200); + if (closed) return; UUID operationId = UUID.fromString(string(task, "operationId")); TaskResult result; synchronized (completed) { @@ -327,10 +365,41 @@ private static int bounded(int value, int min, int max, String name) { @Override public void close() { - closed = true; + Future reload; + CompletableFuture operation; + synchronized (operationLifecycle) { + closed = true; + reload = activeReload; + operation = activeOperation; + } ScheduledFuture current = scheduled; if (current != null) current.cancel(false); - executor.shutdownNow(); + if (reload != null && Bukkit.isPrimaryThread()) reload.cancel(false); + awaitShutdown(executor, operation); + } + + static void awaitShutdown(ScheduledExecutorService executor, CompletableFuture operation) { + executor.shutdown(); + if (operation != null) { + try { + operation.get(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS); + } catch (java.util.concurrent.CancellationException | java.util.concurrent.ExecutionException ignored) { + // A completed failed operation is no longer able to outlive shutdown. + } catch (java.util.concurrent.TimeoutException e) { + throw new IllegalStateException("Bukkit Control operation did not stop before connector shutdown", e); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for the Bukkit Control operation", e); + } + } + try { + if (!executor.awaitTermination(SHUTDOWN_TIMEOUT_SECONDS, TimeUnit.SECONDS)) { + throw new IllegalStateException("Bukkit Control operation did not stop before connector shutdown"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for the Bukkit Control operation", e); + } } private record Settings(String nodeId, URI endpoint, int heartbeatSeconds, int connectTimeoutMillis, diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index eba148e17b..074080fe63 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -175,9 +175,16 @@ void cycle() { onFailure(failure); return; } - activeRequest = primary; CompletableFuture operationDone = new CompletableFuture<>(); activeOperation = operationDone; + if (closed) { + primary.cancel(true); + operationDone.complete(null); + if (activeOperation == operationDone) activeOperation = null; + inFlight.set(false); + return; + } + activeRequest = primary; CompletableFuture operation = primary.thenCompose(response -> { handlePrimaryResponse(response, !registered); registered = true; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index 46aefe56c6..20ca56847d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -5,6 +5,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + import org.junit.jupiter.api.Test; import com.google.gson.JsonArray; @@ -27,4 +31,16 @@ class BackendControlConnectorProtocolTest { assertFalse(BackendControlConnector.negotiatedCapability(explicit, "config.files.v1", true)); assertTrue(BackendControlConnector.negotiatedCapability(explicit, "config.quick-setup.v1", false)); } + + @Test void shutdownWaitsForTheClaimedBackendOperation() throws Exception { + var executor = Executors.newSingleThreadScheduledExecutor(); + CompletableFuture operation = new CompletableFuture<>(); + CompletableFuture closing = CompletableFuture.runAsync( + () -> BackendControlConnector.awaitShutdown(executor, operation)); + + assertThrows(java.util.concurrent.TimeoutException.class, () -> closing.get(100, TimeUnit.MILLISECONDS)); + operation.complete(null); + closing.get(2, TimeUnit.SECONDS); + assertTrue(executor.isTerminated()); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 518d9d4c83..b93097abf9 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -197,6 +197,25 @@ class ControlConnectorTest { } } + @Test void closeBeforeOperationPublicationPreventsTheRequestChainFromStarting() throws Exception { + connector.close(); + transport.firstSendEntered = new CountDownLatch(1); + transport.releaseFirstSend = new CountDownLatch(1); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L); + CompletableFuture cycling = CompletableFuture.runAsync(connector::cycle); + assertTrue(transport.firstSendEntered.await(2, TimeUnit.SECONDS)); + + try { + connector.close(); + } finally { + transport.releaseFirstSend.countDown(); + } + cycling.get(2, TimeUnit.SECONDS); + assertEquals(1, transport.requests.size()); + assertEquals(Status.STOPPED, connector.status()); + } + @Test void backoffIsBoundedExponentialAndJitteredWithoutSleeping() { assertEquals(1000, ControlConnector.backoffMillis(1, 0)); assertEquals(2000, ControlConnector.backoffMillis(2, 0)); @@ -234,6 +253,8 @@ private static final class FakeTransport implements Transport { private RuntimeException synchronousFailure; private boolean acceptConfiguration; private CompletableFuture operationClaim; + private CountDownLatch firstSendEntered; + private CountDownLatch releaseFirstSend; @Override public CompletableFuture send(Request request) { @@ -243,6 +264,17 @@ public CompletableFuture send(Request request) { throw failure; } requests.add(request); + if (firstSendEntered != null) { + firstSendEntered.countDown(); + try { + releaseFirstSend.await(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } finally { + firstSendEntered = null; + } + } if (stalled != null) { CompletableFuture result = stalled; stalled = null; From 3979b62a1318b5d855e2a12e3707b8187592d7f1 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 18:36:21 -0600 Subject: [PATCH 40/73] Bind hosted health to the launched process --- .../proxy/control/HostedControlManager.java | 34 ++++++++------ .../control/HostedControlManagerTest.java | 46 +++++++++++++++---- docs/control-connector.md | 5 +- 3 files changed, 60 insertions(+), 25 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 0e0f935201..790af739a6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -121,8 +121,9 @@ void runOnce() { if (closed) { return; } - ManagedProcess process = launch(settings.jarFile()); - if (awaitHealthy(process)) { + LaunchedProcess launched = launch(settings.jarFile()); + ManagedProcess process = launched.process(); + if (awaitHealthy(launched)) { failures = 0; status = Status.RUNNING; logger.accept("WebUI is available at " + settings.endpoint()); @@ -132,8 +133,9 @@ void runOnce() { stopProcess(process, true); if (updated && Files.isRegularFile(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { rollback(); - ManagedProcess previous = launch(settings.jarFile()); - if (awaitHealthy(previous)) { + LaunchedProcess previousLaunch = launch(settings.jarFile()); + ManagedProcess previous = previousLaunch.process(); + if (awaitHealthy(previousLaunch)) { failures = 0; quarantinedSha256 = settings.sha256(); status = Status.ROLLED_BACK; @@ -219,7 +221,7 @@ private void rollback() throws IOException { atomicMove(settings.previousFile(), settings.jarFile(), false); } - private ManagedProcess launch(Path artifact) throws IOException { + private LaunchedProcess launch(Path artifact) throws IOException { status = Status.STARTING_PROCESS; synchronized (processLifecycle) { if (closed) throw new IOException("Hosted Control manager is closed"); @@ -230,13 +232,15 @@ private ManagedProcess launch(Path artifact) throws IOException { managedProcess = null; exitAwaitedProcess = null; } - ManagedProcess process = launcher.launch(settings, artifact); + String launchId = UUID.randomUUID().toString(); + ManagedProcess process = launcher.launch(settings, artifact, launchId); managedProcess = process; - return process; + return new LaunchedProcess(process, launchId); } } - private boolean awaitHealthy(ManagedProcess process) throws InterruptedException { + private boolean awaitHealthy(LaunchedProcess launched) throws InterruptedException { + ManagedProcess process = launched.process(); long timeoutNanos = TimeUnit.SECONDS.toNanos(settings.startupTimeoutSeconds()); long started = nanoTime.getAsLong(); long deadline = started + timeoutNanos; @@ -255,7 +259,7 @@ private boolean awaitHealthy(ManagedProcess process) throws InterruptedException firstAttempt = false; long remaining = Math.max(1L, deadline - now); if (healthProbe.isHealthy(settings.endpoint(), - Duration.ofNanos(Math.min(TimeUnit.SECONDS.toNanos(2), remaining)))) { + Duration.ofNanos(Math.min(TimeUnit.SECONDS.toNanos(2), remaining)), launched.launchId())) { return process.isAlive(); } if (nanoTime.getAsLong() >= deadline) { @@ -544,11 +548,11 @@ void download(URI source, Path target, long maximumBytes, Duration timeout) } interface ProcessLauncher { - ManagedProcess launch(Settings settings, Path artifact) throws IOException; + ManagedProcess launch(Settings settings, Path artifact, String launchId) throws IOException; } interface HealthProbe { - boolean isHealthy(URI endpoint, Duration timeout); + boolean isHealthy(URI endpoint, Duration timeout, String launchId); } interface Sleeper { @@ -658,7 +662,7 @@ private static Thread bodyTimeoutThread(InputStream input, long deadline, Atomic private static final class JdkProcessLauncher implements ProcessLauncher { @Override - public ManagedProcess launch(Settings settings, Path artifact) throws IOException { + public ManagedProcess launch(Settings settings, Path artifact, String launchId) throws IOException { Path log = artifact.resolveSibling("control-output.log"); if (Files.exists(log, LinkOption.NOFOLLOW_LINKS) && !Files.isRegularFile(log, LinkOption.NOFOLLOW_LINKS)) { @@ -683,6 +687,7 @@ public ManagedProcess launch(Settings settings, Path artifact) throws IOExceptio environment.put("CONTROL_HOST", settings.host()); environment.put("CONTROL_PORT", Integer.toString(settings.port())); environment.put("CONTROL_DATA_DIR", settings.dataDirectory().toString()); + environment.put("CONTROL_LAUNCH_ID", launchId); return new JdkManagedProcess(builder.start()); } @@ -696,7 +701,7 @@ static final class JdkHealthProbe implements HealthProbe { .followRedirects(HttpClient.Redirect.NEVER).build(); @Override - public boolean isHealthy(URI endpoint, Duration timeout) { + public boolean isHealthy(URI endpoint, Duration timeout, String launchId) { try { HttpRequest request = HttpRequest.newBuilder(endpoint.resolve("/api/v1/health")) .timeout(timeout).header("Accept", "application/json").GET().build(); @@ -709,6 +714,7 @@ public boolean isHealthy(URI endpoint, Duration timeout) { JsonObject identity = object.has("identity") && object.get("identity").isJsonObject() ? object.getAsJsonObject("identity") : null; return object.has("status") && "ok".equals(object.get("status").getAsString()) + && object.has("launchId") && launchId.equals(object.get("launchId").getAsString()) && identity != null && identity.has("protocolVersion") && identity.get("protocolVersion").getAsInt() == EXPECTED_PROTOCOL_VERSION; } catch (Exception e) { @@ -717,6 +723,8 @@ public boolean isHealthy(URI endpoint, Duration timeout) { } } + private record LaunchedProcess(ManagedProcess process, String launchId) { } + private static final class JdkManagedProcess implements ManagedProcess { private final Process process; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index c09e64e3ff..f920157bdf 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -101,7 +101,8 @@ void hostedHealthBodyReadCannotOutliveProbeTimeout() throws Exception { try { URI endpoint = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/"); long started = System.nanoTime(); - assertFalse(new HostedControlManager.JdkHealthProbe().isHealthy(endpoint, Duration.ofMillis(250))); + assertFalse(new HostedControlManager.JdkHealthProbe().isHealthy(endpoint, Duration.ofMillis(250), + "00000000-0000-0000-0000-000000000001")); assertTrue(bodyStarted.await(2, TimeUnit.SECONDS)); assertTrue(Duration.ofNanos(System.nanoTime() - started).compareTo(Duration.ofSeconds(2)) < 0); } finally { @@ -110,6 +111,30 @@ void hostedHealthBodyReadCannotOutliveProbeTimeout() throws Exception { } } + @Test + void hostedHealthRequiresTheLaunchedChildIdentity() throws Exception { + String launchId = "00000000-0000-0000-0000-000000000001"; + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/api/v1/health", exchange -> { + byte[] body = ("{\"status\":\"ok\",\"launchId\":\"" + launchId + + "\",\"identity\":{\"protocolVersion\":1}}").getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(200, body.length); + try (var output = exchange.getResponseBody()) { + output.write(body); + } + }); + server.start(); + try { + URI endpoint = URI.create("http://127.0.0.1:" + server.getAddress().getPort() + "/"); + HostedControlManager.JdkHealthProbe probe = new HostedControlManager.JdkHealthProbe(); + assertTrue(probe.isHealthy(endpoint, Duration.ofSeconds(2), launchId)); + assertFalse(probe.isHealthy(endpoint, Duration.ofSeconds(2), + "00000000-0000-0000-0000-000000000002")); + } finally { + server.stop(0); + } + } + @Test void missingPinnedArtifactIsStagedVerifiedAndStarted() throws Exception { byte[] release = "signed-release-content".getBytes(StandardCharsets.UTF_8); @@ -124,7 +149,7 @@ void missingPinnedArtifactIsStagedVerifiedAndStarted() throws Exception { (source, target, maximum, timeout) -> { downloads.incrementAndGet(); Files.write(target, release); - }, launcher, (endpoint, timeout) -> true, + }, launcher, (endpoint, timeout, launchId) -> true, millis -> { throw new AssertionError("healthy startup must not sleep"); }, System::nanoTime, message -> { }); @@ -154,7 +179,7 @@ void failedUpdateRollsBackAndStartsPreviousRelease() throws Exception { ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); HostedControlManager manager = new HostedControlManager(settings, executor, (source, target, maximum, timeout) -> Files.write(target, update), launcher, - (endpoint, timeout) -> healthChecks.incrementAndGet() > 4, + (endpoint, timeout, launchId) -> healthChecks.incrementAndGet() > 4, millis -> clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(millis)), clock::get, message -> { }); manager.runOnce(); @@ -185,7 +210,7 @@ void manualPinnedInstallNeedsNoDownloaderAndCloseIsNonBlocking() throws Exceptio ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); HostedControlManager manager = new HostedControlManager(settings, executor, (source, target, maximum, timeout) -> { throw new AssertionError("manual install downloaded"); }, - launcher, (endpoint, timeout) -> true, millis -> { }, System::nanoTime, message -> { }); + launcher, (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, message -> { }); manager.runOnce(); manager.close(); @@ -206,7 +231,7 @@ void digestMismatchNeverActivatesOrLaunchesDownloadedCode() throws Exception { ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); HostedControlManager manager = new HostedControlManager(settings, executor, (source, target, maximum, timeout) -> Files.write(target, untrusted), launcher, - (endpoint, timeout) -> true, millis -> { }, System::nanoTime, message -> { }); + (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, message -> { }); manager.runOnce(); @@ -234,7 +259,7 @@ void closeAndWaitDoesNotReturnUntilArtifactWorkerStops() throws Exception { } catch (InterruptedException ignored) { } } Files.write(target, release); - }, new FakeLauncher(), (endpoint, timeout) -> true, millis -> { }, System::nanoTime, message -> { }); + }, new FakeLauncher(), (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, message -> { }); manager.start(); assertTrue(entered.await(2, TimeUnit.SECONDS)); CountDownLatch closingStarted = new CountDownLatch(1); @@ -266,7 +291,7 @@ void failedForcedTerminationRetainsManagedProcessForRetry() throws Exception { HostedControlManager manager = new HostedControlManager( settings(root.resolve("control.jar"), root.resolve("data"), false, false, digest("x".getBytes()), 1), executor, (source, target, maximum, timeout) -> { }, new FakeLauncher(), - (endpoint, timeout) -> true, millis -> { }, System::nanoTime, message -> { }); + (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, message -> { }); UnkillableProcess process = new UnkillableProcess(); java.lang.reflect.Field managed = HostedControlManager.class.getDeclaredField("managedProcess"); managed.setAccessible(true); @@ -291,10 +316,10 @@ void failedHealthTerminationDefersRetryUntilRetainedProcessExits() throws Except AtomicLong clock = new AtomicLong(); HostedControlManager manager = new HostedControlManager( settings(jar, root.resolve("data"), false, false, digest(release), 1), executor, - (source, target, maximum, timeout) -> { }, (settings, artifact) -> { + (source, target, maximum, timeout) -> { }, (settings, artifact, launchId) -> { launches.incrementAndGet(); return process; - }, (endpoint, timeout) -> false, + }, (endpoint, timeout, launchId) -> false, millis -> clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(millis)), clock::get, message -> { }); manager.runOnce(); @@ -327,7 +352,8 @@ private static final class FakeLauncher implements HostedControlManager.ProcessL private final List processes = new ArrayList<>(); @Override - public HostedControlManager.ManagedProcess launch(HostedControlManager.Settings settings, Path artifact) + public HostedControlManager.ManagedProcess launch(HostedControlManager.Settings settings, Path artifact, + String launchId) throws java.io.IOException { launchedContents.add(Files.readString(artifact)); FakeProcess process = new FakeProcess(); diff --git a/docs/control-connector.md b/docs/control-connector.md index 395f6e4e3c..11fc8c4369 100644 --- a/docs/control-connector.md +++ b/docs/control-connector.md @@ -59,8 +59,9 @@ without downloading. `AutoUpdate` is false by default; when explicitly enabled a current release is retained as `.previous`. A failed process/protocol health check atomically restores and starts that previous release, retaining the failed candidate as `.failed`. Unexpected exits use bounded restart backoff. -The child receives only its bind host, port, and contained data directory. It does not receive VotingPlugin configuration -or credentials in release metadata. Process output is written beside the hosted JAR and rotated at 1 MiB. Configure the +The child receives only its bind host, port, contained data directory, and a random per-launch ID. Health must echo that +ID, preventing an unrelated Control process on the same port from satisfying startup checks. It does not receive +VotingPlugin configuration or credentials in release metadata. Process output is written beside the hosted JAR and rotated at 1 MiB. Configure the WebUI password directly against the hosted data directory (the prompt is not echoed and the password is not an argument): ```shell From de5331319e532ec2b05be1bab462999c1b514f16 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 18:43:24 -0600 Subject: [PATCH 41/73] Validate full backend transport configuration --- .../control/BackendConfigurationService.java | 12 ++++++++++-- .../control/BackendConfigurationServiceTest.java | 12 ++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index f6c48d28b7..82057fd926 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -133,7 +133,11 @@ private static void copyBackupNoFollow(Path backup, Path target, String expected private Preview preview(String fileName, String proposedContent, String current) { YamlConfiguration currentYaml = parse(current); - String resolved = resolveSecrets(parse(proposedContent), currentYaml).saveToString(); + YamlConfiguration resolvedYaml = resolveSecrets(parse(proposedContent), currentYaml); + if ("BungeeSettings.yml".equals(fileName) && resolvedYaml.contains("BungeeMethod")) { + resolvedYaml.set("BungeeMethod", canonicalBungeeMethod(resolvedYaml.getString("BungeeMethod"))); + } + String resolved = resolvedYaml.saveToString(); ensureBounded(resolved); return new Preview(fileName, resolved, revision(current), changes(currentYaml, parse(resolved))); } @@ -342,10 +346,14 @@ private static void move(Path source, Path target) throws IOException { private static String bungeeMethodOption(Map options) { String value = options == null ? "PLUGINMESSAGING" : options.getOrDefault("method", "PLUGINMESSAGING"); + return canonicalBungeeMethod(value); + } + + private static String canonicalBungeeMethod(String value) { for (BungeeMethod method : BungeeMethod.values()) { if (method.name().equalsIgnoreCase(value)) return method.name(); } - throw new IllegalArgumentException("quick setup option method is invalid"); + throw new IllegalArgumentException("BungeeMethod is invalid"); } private static String option(Map options, String name, String pattern) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 111a1f8a5b..b8fb938331 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -170,4 +170,16 @@ class BackendConfigurationServiceTest { Map.of("server", "lobby", "method", "mqtt")); assertTrue(mqtt.proposal().content().contains("BungeeMethod: MQTT")); } + + @Test void fullBungeeSettingsRejectsUnknownTransportMethods() throws Exception { + Files.writeString(directory.resolve("BungeeSettings.yml"), + "UseBungeecord: true\nServer: lobby\nBungeeMethod: PLUGINMESSAGING\n"); + BackendConfigurationService service = new BackendConfigurationService(directory, () -> { }); + + assertThrows(IllegalArgumentException.class, () -> service.preview("BungeeSettings.yml", + "UseBungeecord: true\nServer: lobby\nBungeeMethod: NOT_A_TRANSPORT\n")); + BackendConfigurationService.Preview mqtt = service.preview("BungeeSettings.yml", + "UseBungeecord: true\nServer: lobby\nBungeeMethod: mqtt\n"); + assertTrue(mqtt.resolvedContent().contains("BungeeMethod: MQTT")); + } } From 0b330f59b7d61ba06f24d9730a39c57d3b41b546 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 18:56:19 -0600 Subject: [PATCH 42/73] Preserve recoverable Control startup state --- .../votingplugin/VotingPluginMain.java | 26 ++++++++++---- .../proxy/control/HostedControlManager.java | 6 +++- .../control/HostedControlManagerTest.java | 34 +++++++++++++++++++ 3 files changed, 59 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 6808fc6808..0444184200 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -442,8 +442,20 @@ public boolean isVoteSite(String voteSite) { } private void loadBungeeHandler() { - backendProxyHandler = new BackendProxyHandler(this, backendProcessedVoteCache); - backendProxyHandler.load(); + BackendProxyHandler candidate = new BackendProxyHandler(this, backendProcessedVoteCache); + try { + candidate.load(); + backendProxyHandler = candidate; + } catch (RuntimeException failure) { + try { + candidate.close(); + } catch (RuntimeException closeFailure) { + failure.addSuppressed(closeFailure); + } + backendProxyHandler = null; + getLogger().warning("Backend proxy transport was not started; it will be retried on reload: " + + failure.getMessage()); + } if (getOptions().getServer().equalsIgnoreCase("PleaseSet")) { getLogger().warning("Bungeecoord is true and server name is not set, bungeecoord features may not work"); @@ -1034,13 +1046,15 @@ private void reloadPlugin(boolean userStorage) { reloadAdvancedCore(userStorage); if (bungeeSettings.isUseBungeecoord()) { - if (getBackendProxyHandler() == null) { + BackendProxyHandler handler = getBackendProxyHandler(); + if (handler == null) { loadBungeeHandler(); + handler = getBackendProxyHandler(); } else { - getBackendProxyHandler().reloadPresenceReporting(); + handler.reloadPresenceReporting(); } - if (userStorage) { - getBackendProxyHandler().loadGlobalMysql(); + if (userStorage && handler != null) { + handler.loadGlobalMysql(); } } else if (getBackendProxyHandler() != null) { getBackendProxyHandler().disablePresenceReporting(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 790af739a6..be32e48da3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -65,6 +65,7 @@ public final class HostedControlManager implements AutoCloseable { private volatile int failures; private volatile String quarantinedSha256; private volatile String trustedRollbackSha256; + private volatile boolean rollbackPending; private HostedControlManager(Settings settings, Consumer logger) { this(settings, daemonExecutor(), new JdkArtifactDownloader(), new JdkProcessLauncher(), @@ -118,6 +119,7 @@ void runOnce() { clearManagedProcess(retained); } boolean updated = prepareArtifact(); + if (updated) rollbackPending = true; if (closed) { return; } @@ -125,14 +127,16 @@ void runOnce() { ManagedProcess process = launched.process(); if (awaitHealthy(launched)) { failures = 0; + rollbackPending = false; status = Status.RUNNING; logger.accept("WebUI is available at " + settings.endpoint()); monitor(process); return; } stopProcess(process, true); - if (updated && Files.isRegularFile(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { + if (rollbackPending && Files.isRegularFile(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { rollback(); + rollbackPending = false; LaunchedProcess previousLaunch = launch(settings.jarFile()); ManagedProcess previous = previousLaunch.process(); if (awaitHealthy(previousLaunch)) { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index f920157bdf..d2a365f6c2 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -333,6 +333,40 @@ void failedHealthTerminationDefersRetryUntilRetainedProcessExits() throws Except manager.close(); } + @Test + void delayedUpdateProcessExitPreservesRollbackEligibility() throws Exception { + byte[] previous = "previous-release".getBytes(StandardCharsets.UTF_8); + byte[] update = "bad-new-release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control.jar"); + Files.write(jar, previous); + HostedControlManager.Settings settings = settings(jar, root.resolve("data"), true, true, + digest(update), 1); + ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1); + RetainedProcess retained = new RetainedProcess(); + List launches = new ArrayList<>(); + AtomicInteger launchCount = new AtomicInteger(); + AtomicLong clock = new AtomicLong(); + HostedControlManager manager = new HostedControlManager(settings, executor, + (source, target, maximum, timeout) -> Files.write(target, update), + (launchSettings, artifact, launchId) -> { + launches.add(Files.readString(artifact)); + return launchCount.getAndIncrement() == 0 ? retained : new FakeProcess(); + }, (endpoint, timeout, launchId) -> "previous-release".equals(launches.get(launches.size() - 1)), + millis -> clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(millis)), clock::get, message -> { }); + + manager.runOnce(); + assertEquals(HostedControlManager.Status.FAILED, manager.status()); + retained.exit(); + manager.runOnce(); + + assertEquals(HostedControlManager.Status.ROLLED_BACK, manager.status()); + assertEquals(List.of("bad-new-release", "bad-new-release", "previous-release"), launches); + assertEquals("previous-release", Files.readString(jar)); + assertEquals("bad-new-release", Files.readString(settings.failedFile())); + manager.close(); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); From 786398161f5d025462f37d1e0daf008557d11e40 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 19:07:23 -0600 Subject: [PATCH 43/73] Rollback retained Control updates before retry --- .../proxy/control/HostedControlManager.java | 35 ++++++++++++------- .../control/HostedControlManagerTest.java | 2 +- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index be32e48da3..0fd06c97ca 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -118,6 +118,10 @@ void runOnce() { if (retained.isAlive()) return; clearManagedProcess(retained); } + if (rollbackPending && Files.isRegularFile(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { + if (rollbackAndStartPrevious()) return; + throw new IOException("The previous Control release did not become healthy"); + } boolean updated = prepareArtifact(); if (updated) rollbackPending = true; if (closed) { @@ -135,19 +139,7 @@ void runOnce() { } stopProcess(process, true); if (rollbackPending && Files.isRegularFile(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { - rollback(); - rollbackPending = false; - LaunchedProcess previousLaunch = launch(settings.jarFile()); - ManagedProcess previous = previousLaunch.process(); - if (awaitHealthy(previousLaunch)) { - failures = 0; - quarantinedSha256 = settings.sha256(); - status = Status.ROLLED_BACK; - logger.accept("The new Control release failed health checks; the previous release is running"); - monitor(previous); - return; - } - stopProcess(previous, true); + if (rollbackAndStartPrevious()) return; } throw new IOException("Control did not become healthy"); } catch (InterruptedException e) { @@ -164,6 +156,23 @@ void runOnce() { } } + private boolean rollbackAndStartPrevious() throws IOException, InterruptedException { + rollback(); + rollbackPending = false; + LaunchedProcess previousLaunch = launch(settings.jarFile()); + ManagedProcess previous = previousLaunch.process(); + if (awaitHealthy(previousLaunch)) { + failures = 0; + quarantinedSha256 = settings.sha256(); + status = Status.ROLLED_BACK; + logger.accept("The new Control release failed health checks; the previous release is running"); + monitor(previous); + return true; + } + stopProcess(previous, true); + return false; + } + private boolean prepareArtifact() throws IOException, InterruptedException { secureDirectory(settings.rootDirectory(), settings.jarFile().getParent()); secureDirectory(settings.rootDirectory(), settings.dataDirectory()); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index d2a365f6c2..6e0faacf38 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -361,7 +361,7 @@ void delayedUpdateProcessExitPreservesRollbackEligibility() throws Exception { manager.runOnce(); assertEquals(HostedControlManager.Status.ROLLED_BACK, manager.status()); - assertEquals(List.of("bad-new-release", "bad-new-release", "previous-release"), launches); + assertEquals(List.of("bad-new-release", "previous-release"), launches); assertEquals("previous-release", Files.readString(jar)); assertEquals("bad-new-release", Files.readString(settings.failedFile())); manager.close(); From 0d66ad1944e89b0653b15b4f4ad3c3a746f04f12 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 19:17:45 -0600 Subject: [PATCH 44/73] Persist hosted Control rollback intent --- .../proxy/control/HostedControlManager.java | 88 +++++++++++++++++-- .../control/HostedControlManagerTest.java | 36 ++++++++ 2 files changed, 119 insertions(+), 5 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 0fd06c97ca..7a178ac77d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -7,6 +7,8 @@ import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.LinkOption; @@ -113,6 +115,9 @@ void runOnce() { return; } try { + secureDirectory(settings.rootDirectory(), settings.jarFile().getParent()); + secureDirectory(settings.rootDirectory(), settings.dataDirectory()); + recoverPersistedRollbackState(); ManagedProcess retained = managedProcess; if (retained != null) { if (retained.isAlive()) return; @@ -131,6 +136,7 @@ void runOnce() { ManagedProcess process = launched.process(); if (awaitHealthy(launched)) { failures = 0; + clearPersistedRollbackState(); rollbackPending = false; status = Status.RUNNING; logger.accept("WebUI is available at " + settings.endpoint()); @@ -174,10 +180,9 @@ private boolean rollbackAndStartPrevious() throws IOException, InterruptedExcept } private boolean prepareArtifact() throws IOException, InterruptedException { - secureDirectory(settings.rootDirectory(), settings.jarFile().getParent()); - secureDirectory(settings.rootDirectory(), settings.dataDirectory()); + String installedSha256 = null; if (Files.isRegularFile(settings.jarFile(), LinkOption.NOFOLLOW_LINKS)) { - String installedSha256 = sha256(settings.jarFile()); + installedSha256 = sha256(settings.jarFile()); if (settings.sha256().equals(installedSha256)) { return false; } @@ -206,15 +211,16 @@ private boolean prepareArtifact() throws IOException, InterruptedException { if (!settings.sha256().equals(sha256(staged))) { throw new IOException("Downloaded Control artifact failed SHA-256 verification"); } - activate(staged, installed); + activate(staged, installed, installedSha256); return installed; } finally { Files.deleteIfExists(staged); } } - private void activate(Path staged, boolean installed) throws IOException { + private void activate(Path staged, boolean installed, String installedSha256) throws IOException { if (installed) { + persistRollbackState(installedSha256); atomicMove(settings.jarFile(), settings.previousFile(), true); } try { @@ -222,6 +228,7 @@ private void activate(Path staged, boolean installed) throws IOException { } catch (IOException e) { if (installed && Files.exists(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { atomicMove(settings.previousFile(), settings.jarFile(), true); + clearPersistedRollbackState(); } throw e; } @@ -232,6 +239,73 @@ private void rollback() throws IOException { atomicMove(settings.jarFile(), settings.failedFile(), true); } atomicMove(settings.previousFile(), settings.jarFile(), false); + clearPersistedRollbackState(); + } + + private void persistRollbackState(String previousSha256) throws IOException { + if (previousSha256 == null || !previousSha256.matches("[0-9a-f]{64}")) { + throw new IOException("Control rollback digest is unavailable"); + } + Path marker = settings.rollbackPendingFile(); + if (Files.exists(marker, LinkOption.NOFOLLOW_LINKS) + && !Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Control rollback state path is not a regular file"); + } + Path staged = marker.resolveSibling(marker.getFileName() + ".staged-" + UUID.randomUUID()); + byte[] state = (previousSha256 + "\n" + settings.sha256() + "\n") + .getBytes(java.nio.charset.StandardCharsets.US_ASCII); + try (FileChannel channel = FileChannel.open(staged, StandardOpenOption.CREATE_NEW, + StandardOpenOption.WRITE)) { + ByteBuffer buffer = ByteBuffer.wrap(state); + while (buffer.hasRemaining()) channel.write(buffer); + channel.force(true); + } + try { + atomicMove(staged, marker, true); + } finally { + Files.deleteIfExists(staged); + } + } + + private void recoverPersistedRollbackState() throws IOException { + Path marker = settings.rollbackPendingFile(); + if (!Files.exists(marker, LinkOption.NOFOLLOW_LINKS)) return; + if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Control rollback state is invalid"); + } + byte[] bytes; + try (FileChannel channel = FileChannel.open(marker, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { + long size = channel.size(); + if (size > 256L) throw new IOException("Control rollback state is invalid"); + bytes = new byte[(int) size]; + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining() && channel.read(buffer) >= 0) { } + } + String[] lines = new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).split("\\R"); + if (lines.length != 2 || !lines[0].matches("[0-9a-f]{64}") || !lines[1].matches("[0-9a-f]{64}")) { + throw new IOException("Control rollback state is invalid"); + } + String previousSha256 = lines[0]; + String candidateSha256 = lines[1]; + boolean active = Files.isRegularFile(settings.jarFile(), LinkOption.NOFOLLOW_LINKS); + String activeSha256 = active ? sha256(settings.jarFile()) : null; + if (previousSha256.equals(activeSha256)) { + clearPersistedRollbackState(); + return; + } + if (active && !candidateSha256.equals(activeSha256)) { + throw new IOException("Active Control artifact does not match pending rollback state"); + } + if (!Files.isRegularFile(settings.previousFile(), LinkOption.NOFOLLOW_LINKS) + || !previousSha256.equals(sha256(settings.previousFile()))) { + throw new IOException("Previous Control artifact does not match pending rollback state"); + } + trustedRollbackSha256 = previousSha256; + rollbackPending = true; + } + + private void clearPersistedRollbackState() throws IOException { + Files.deleteIfExists(settings.rollbackPendingFile()); } private LaunchedProcess launch(Path artifact) throws IOException { @@ -546,6 +620,10 @@ Path failedFile() { return jarFile.resolveSibling(jarFile.getFileName() + ".failed"); } + Path rollbackPendingFile() { + return jarFile.resolveSibling(jarFile.getFileName() + ".rollback-pending"); + } + URI endpoint() { try { return new URI("http", null, host, port, "/", null, null); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 6e0faacf38..be155df76c 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -367,6 +367,42 @@ void delayedUpdateProcessExitPreservesRollbackEligibility() throws Exception { manager.close(); } + @Test + void pendingUpdateRollbackSurvivesManagerRestart() throws Exception { + byte[] previous = "previous-release".getBytes(StandardCharsets.UTF_8); + byte[] update = "bad-new-release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control.jar"); + Files.write(jar, previous); + HostedControlManager.Settings settings = settings(jar, root.resolve("data"), true, true, + digest(update), 1); + FakeLauncher failedLauncher = new FakeLauncher(); + ScheduledExecutorService failedExecutor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager failedManager = new HostedControlManager(settings, failedExecutor, + (source, target, maximum, timeout) -> Files.write(target, update), failedLauncher, + (endpoint, timeout, launchId) -> { throw new IllegalStateException("simulated JVM exit"); }, + millis -> { }, System::nanoTime, message -> { }); + + failedManager.runOnce(); + failedExecutor.shutdownNow(); + assertEquals("bad-new-release", Files.readString(jar)); + assertTrue(Files.isRegularFile(settings.rollbackPendingFile())); + + FakeLauncher recoveredLauncher = new FakeLauncher(); + ScheduledExecutorService recoveredExecutor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager recoveredManager = new HostedControlManager(settings, recoveredExecutor, + (source, target, maximum, timeout) -> { throw new AssertionError("recovery downloaded"); }, + recoveredLauncher, (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, + message -> { }); + recoveredManager.runOnce(); + + assertEquals(HostedControlManager.Status.ROLLED_BACK, recoveredManager.status()); + assertEquals(List.of("previous-release"), recoveredLauncher.launchedContents); + assertEquals("previous-release", Files.readString(jar)); + assertFalse(Files.exists(settings.rollbackPendingFile())); + recoveredManager.close(); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); From 4b2d68479b208662b10d3b4cdad228f985109a1b Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 19:27:54 -0600 Subject: [PATCH 45/73] Quarantine the persisted Control candidate --- .../proxy/control/HostedControlManager.java | 13 +++++++-- .../control/HostedControlManagerTest.java | 29 ++++++++++++++----- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 7a178ac77d..f1d2f56a35 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -67,6 +67,7 @@ public final class HostedControlManager implements AutoCloseable { private volatile int failures; private volatile String quarantinedSha256; private volatile String trustedRollbackSha256; + private volatile String rollbackCandidateSha256; private volatile boolean rollbackPending; private HostedControlManager(Settings settings, Consumer logger) { @@ -128,7 +129,10 @@ void runOnce() { throw new IOException("The previous Control release did not become healthy"); } boolean updated = prepareArtifact(); - if (updated) rollbackPending = true; + if (updated) { + rollbackCandidateSha256 = settings.sha256(); + rollbackPending = true; + } if (closed) { return; } @@ -137,6 +141,7 @@ void runOnce() { if (awaitHealthy(launched)) { failures = 0; clearPersistedRollbackState(); + rollbackCandidateSha256 = null; rollbackPending = false; status = Status.RUNNING; logger.accept("WebUI is available at " + settings.endpoint()); @@ -169,7 +174,8 @@ private boolean rollbackAndStartPrevious() throws IOException, InterruptedExcept ManagedProcess previous = previousLaunch.process(); if (awaitHealthy(previousLaunch)) { failures = 0; - quarantinedSha256 = settings.sha256(); + quarantinedSha256 = rollbackCandidateSha256; + rollbackCandidateSha256 = null; status = Status.ROLLED_BACK; logger.accept("The new Control release failed health checks; the previous release is running"); monitor(previous); @@ -291,6 +297,8 @@ private void recoverPersistedRollbackState() throws IOException { String activeSha256 = active ? sha256(settings.jarFile()) : null; if (previousSha256.equals(activeSha256)) { clearPersistedRollbackState(); + rollbackCandidateSha256 = null; + rollbackPending = false; return; } if (active && !candidateSha256.equals(activeSha256)) { @@ -301,6 +309,7 @@ private void recoverPersistedRollbackState() throws IOException { throw new IOException("Previous Control artifact does not match pending rollback state"); } trustedRollbackSha256 = previousSha256; + rollbackCandidateSha256 = candidateSha256; rollbackPending = true; } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index be155df76c..604ef88522 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -370,28 +370,35 @@ void delayedUpdateProcessExitPreservesRollbackEligibility() throws Exception { @Test void pendingUpdateRollbackSurvivesManagerRestart() throws Exception { byte[] previous = "previous-release".getBytes(StandardCharsets.UTF_8); - byte[] update = "bad-new-release".getBytes(StandardCharsets.UTF_8); + byte[] failedUpdate = "bad-new-release-a".getBytes(StandardCharsets.UTF_8); + byte[] nextUpdate = "new-release-b".getBytes(StandardCharsets.UTF_8); Path root = directory.toAbsolutePath().normalize(); Path jar = root.resolve("control.jar"); Files.write(jar, previous); HostedControlManager.Settings settings = settings(jar, root.resolve("data"), true, true, - digest(update), 1); + digest(failedUpdate), 1); FakeLauncher failedLauncher = new FakeLauncher(); ScheduledExecutorService failedExecutor = Executors.newSingleThreadScheduledExecutor(); HostedControlManager failedManager = new HostedControlManager(settings, failedExecutor, - (source, target, maximum, timeout) -> Files.write(target, update), failedLauncher, + (source, target, maximum, timeout) -> Files.write(target, failedUpdate), failedLauncher, (endpoint, timeout, launchId) -> { throw new IllegalStateException("simulated JVM exit"); }, millis -> { }, System::nanoTime, message -> { }); failedManager.runOnce(); failedExecutor.shutdownNow(); - assertEquals("bad-new-release", Files.readString(jar)); + assertEquals("bad-new-release-a", Files.readString(jar)); assertTrue(Files.isRegularFile(settings.rollbackPendingFile())); + HostedControlManager.Settings recoveredSettings = settings(jar, root.resolve("data"), true, true, + digest(nextUpdate), 1); FakeLauncher recoveredLauncher = new FakeLauncher(); + AtomicInteger recoveredDownloads = new AtomicInteger(); ScheduledExecutorService recoveredExecutor = Executors.newSingleThreadScheduledExecutor(); - HostedControlManager recoveredManager = new HostedControlManager(settings, recoveredExecutor, - (source, target, maximum, timeout) -> { throw new AssertionError("recovery downloaded"); }, + HostedControlManager recoveredManager = new HostedControlManager(recoveredSettings, recoveredExecutor, + (source, target, maximum, timeout) -> { + recoveredDownloads.incrementAndGet(); + Files.write(target, nextUpdate); + }, recoveredLauncher, (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, message -> { }); recoveredManager.runOnce(); @@ -399,7 +406,15 @@ void pendingUpdateRollbackSurvivesManagerRestart() throws Exception { assertEquals(HostedControlManager.Status.ROLLED_BACK, recoveredManager.status()); assertEquals(List.of("previous-release"), recoveredLauncher.launchedContents); assertEquals("previous-release", Files.readString(jar)); - assertFalse(Files.exists(settings.rollbackPendingFile())); + assertEquals(0, recoveredDownloads.get()); + assertFalse(Files.exists(recoveredSettings.rollbackPendingFile())); + + recoveredLauncher.processes.get(0).destroy(); + recoveredManager.runOnce(); + assertEquals(HostedControlManager.Status.RUNNING, recoveredManager.status()); + assertEquals(1, recoveredDownloads.get()); + assertEquals(List.of("previous-release", "new-release-b"), recoveredLauncher.launchedContents); + assertEquals("new-release-b", Files.readString(jar)); recoveredManager.close(); } From fa635c1c1c79e39cdcb0d297da7d0485f6576100 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 19:38:08 -0600 Subject: [PATCH 46/73] Persist hosted Control update quarantine --- .../proxy/control/HostedControlManager.java | 91 ++++++++++++++----- .../control/HostedControlManagerTest.java | 36 ++++++++ 2 files changed, 105 insertions(+), 22 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index f1d2f56a35..4eaa0dd842 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -128,6 +128,7 @@ void runOnce() { if (rollbackAndStartPrevious()) return; throw new IOException("The previous Control release did not become healthy"); } + recoverPersistedQuarantineState(); boolean updated = prepareArtifact(); if (updated) { rollbackCandidateSha256 = settings.sha256(); @@ -168,14 +169,16 @@ void runOnce() { } private boolean rollbackAndStartPrevious() throws IOException, InterruptedException { + String failedCandidateSha256 = rollbackCandidateSha256; + persistQuarantineState(trustedRollbackSha256, failedCandidateSha256); rollback(); rollbackPending = false; + quarantinedSha256 = failedCandidateSha256; + rollbackCandidateSha256 = null; LaunchedProcess previousLaunch = launch(settings.jarFile()); ManagedProcess previous = previousLaunch.process(); if (awaitHealthy(previousLaunch)) { failures = 0; - quarantinedSha256 = rollbackCandidateSha256; - rollbackCandidateSha256 = null; status = Status.ROLLED_BACK; logger.accept("The new Control release failed health checks; the previous release is running"); monitor(previous); @@ -249,16 +252,27 @@ private void rollback() throws IOException { } private void persistRollbackState(String previousSha256) throws IOException { + persistDigestState(settings.rollbackPendingFile(), previousSha256, settings.sha256(), "rollback"); + } + + private void persistQuarantineState(String previousSha256, String candidateSha256) throws IOException { + persistDigestState(settings.quarantineFile(), previousSha256, candidateSha256, "quarantine"); + } + + private void persistDigestState(Path marker, String previousSha256, String candidateSha256, String name) + throws IOException { if (previousSha256 == null || !previousSha256.matches("[0-9a-f]{64}")) { - throw new IOException("Control rollback digest is unavailable"); + throw new IOException("Control " + name + " digest is unavailable"); + } + if (candidateSha256 == null || !candidateSha256.matches("[0-9a-f]{64}")) { + throw new IOException("Control " + name + " candidate digest is unavailable"); } - Path marker = settings.rollbackPendingFile(); if (Files.exists(marker, LinkOption.NOFOLLOW_LINKS) && !Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS)) { - throw new IOException("Control rollback state path is not a regular file"); + throw new IOException("Control " + name + " state path is not a regular file"); } Path staged = marker.resolveSibling(marker.getFileName() + ".staged-" + UUID.randomUUID()); - byte[] state = (previousSha256 + "\n" + settings.sha256() + "\n") + byte[] state = (previousSha256 + "\n" + candidateSha256 + "\n") .getBytes(java.nio.charset.StandardCharsets.US_ASCII); try (FileChannel channel = FileChannel.open(staged, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE)) { @@ -275,22 +289,8 @@ private void persistRollbackState(String previousSha256) throws IOException { private void recoverPersistedRollbackState() throws IOException { Path marker = settings.rollbackPendingFile(); - if (!Files.exists(marker, LinkOption.NOFOLLOW_LINKS)) return; - if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS)) { - throw new IOException("Control rollback state is invalid"); - } - byte[] bytes; - try (FileChannel channel = FileChannel.open(marker, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { - long size = channel.size(); - if (size > 256L) throw new IOException("Control rollback state is invalid"); - bytes = new byte[(int) size]; - ByteBuffer buffer = ByteBuffer.wrap(bytes); - while (buffer.hasRemaining() && channel.read(buffer) >= 0) { } - } - String[] lines = new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).split("\\R"); - if (lines.length != 2 || !lines[0].matches("[0-9a-f]{64}") || !lines[1].matches("[0-9a-f]{64}")) { - throw new IOException("Control rollback state is invalid"); - } + String[] lines = readDigestState(marker, "rollback"); + if (lines == null) return; String previousSha256 = lines[0]; String candidateSha256 = lines[1]; boolean active = Files.isRegularFile(settings.jarFile(), LinkOption.NOFOLLOW_LINKS); @@ -313,10 +313,53 @@ private void recoverPersistedRollbackState() throws IOException { rollbackPending = true; } + private void recoverPersistedQuarantineState() throws IOException { + String[] lines = readDigestState(settings.quarantineFile(), "quarantine"); + if (lines == null) return; + String previousSha256 = lines[0]; + String candidateSha256 = lines[1]; + if (!settings.sha256().equals(candidateSha256)) { + clearPersistedQuarantineState(); + quarantinedSha256 = null; + trustedRollbackSha256 = null; + return; + } + if (!Files.isRegularFile(settings.jarFile(), LinkOption.NOFOLLOW_LINKS) + || !previousSha256.equals(sha256(settings.jarFile()))) { + throw new IOException("Active Control artifact does not match quarantined rollback state"); + } + trustedRollbackSha256 = previousSha256; + quarantinedSha256 = candidateSha256; + } + + private String[] readDigestState(Path marker, String name) throws IOException { + if (!Files.exists(marker, LinkOption.NOFOLLOW_LINKS)) return null; + if (!Files.isRegularFile(marker, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Control " + name + " state is invalid"); + } + byte[] bytes; + try (FileChannel channel = FileChannel.open(marker, StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS)) { + long size = channel.size(); + if (size > 256L) throw new IOException("Control " + name + " state is invalid"); + bytes = new byte[(int) size]; + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining() && channel.read(buffer) >= 0) { } + } + String[] lines = new String(bytes, java.nio.charset.StandardCharsets.US_ASCII).split("\\R"); + if (lines.length != 2 || !lines[0].matches("[0-9a-f]{64}") || !lines[1].matches("[0-9a-f]{64}")) { + throw new IOException("Control " + name + " state is invalid"); + } + return lines; + } + private void clearPersistedRollbackState() throws IOException { Files.deleteIfExists(settings.rollbackPendingFile()); } + private void clearPersistedQuarantineState() throws IOException { + Files.deleteIfExists(settings.quarantineFile()); + } + private LaunchedProcess launch(Path artifact) throws IOException { status = Status.STARTING_PROCESS; synchronized (processLifecycle) { @@ -633,6 +676,10 @@ Path rollbackPendingFile() { return jarFile.resolveSibling(jarFile.getFileName() + ".rollback-pending"); } + Path quarantineFile() { + return jarFile.resolveSibling(jarFile.getFileName() + ".quarantined"); + } + URI endpoint() { try { return new URI("http", null, host, port, "/", null, null); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 604ef88522..8064e98194 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -418,6 +418,42 @@ void pendingUpdateRollbackSurvivesManagerRestart() throws Exception { recoveredManager.close(); } + @Test + void quarantinedUpdateDigestSurvivesManagerRestart() throws Exception { + byte[] previous = "previous-release".getBytes(StandardCharsets.UTF_8); + byte[] update = "bad-new-release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control.jar"); + Files.write(jar, previous); + HostedControlManager.Settings settings = settings(jar, root.resolve("data"), true, true, + digest(update), 1); + FakeLauncher failedLauncher = new FakeLauncher(); + AtomicInteger healthChecks = new AtomicInteger(); + AtomicLong clock = new AtomicLong(); + ScheduledExecutorService failedExecutor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager failedManager = new HostedControlManager(settings, failedExecutor, + (source, target, maximum, timeout) -> Files.write(target, update), failedLauncher, + (endpoint, timeout, launchId) -> healthChecks.incrementAndGet() > 4, + millis -> clock.addAndGet(TimeUnit.MILLISECONDS.toNanos(millis)), clock::get, message -> { }); + failedManager.runOnce(); + assertEquals(HostedControlManager.Status.ROLLED_BACK, failedManager.status()); + assertTrue(Files.isRegularFile(settings.quarantineFile())); + failedManager.close(); + + FakeLauncher restartedLauncher = new FakeLauncher(); + ScheduledExecutorService restartedExecutor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager restartedManager = new HostedControlManager(settings, restartedExecutor, + (source, target, maximum, timeout) -> { throw new AssertionError("quarantined update downloaded"); }, + restartedLauncher, (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, + message -> { }); + restartedManager.runOnce(); + + assertEquals(HostedControlManager.Status.RUNNING, restartedManager.status()); + assertEquals(List.of("previous-release"), restartedLauncher.launchedContents); + assertEquals("previous-release", Files.readString(jar)); + restartedManager.close(); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); From 255c700311fa398207220b3e1de7a14aaac1424d Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 19:51:29 -0600 Subject: [PATCH 47/73] Close Control apply handoff races --- .../backendproxy/BackendProxyHandler.java | 2 +- .../transport/RedisBackendProxyTransport.java | 44 +++++++++++++------ .../control/BackendConfigurationService.java | 9 ++++ .../proxy/velocity/VelocityConfig.java | 2 +- .../BackendProxyHandlerLifecycleTest.java | 22 +++++++--- .../BackendConfigurationServiceTest.java | 16 +++++++ .../velocity/VelocityConfigControlTest.java | 34 ++++++++++++++ 7 files changed, 107 insertions(+), 22 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfigControlTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 3a18d1a76c..966b989c3a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -103,7 +103,7 @@ public void close() { /** Releases a same-method subscriber/listener before its replacement starts. */ public void prepareForReplacement(BungeeMethod replacementMethod) { - if (method == replacementMethod && method != BungeeMethod.PLUGINMESSAGING) { + if (method == replacementMethod && method != BungeeMethod.PLUGINMESSAGING && method != BungeeMethod.REDIS) { transportManager.prepareForReplacement(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index c2dc2bb2df..89d50e2005 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -1,5 +1,8 @@ package com.bencodez.votingplugin.backendproxy.transport; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; @@ -17,7 +20,7 @@ public class RedisBackendProxyTransport implements BackendProxyTransport { private final VotingPluginMain plugin; @Getter private RedisHandler redisHandler; - private Thread listenerThread; + private CountDownLatch subscriptionReady; public RedisBackendProxyTransport(VotingPluginMain plugin) { this.plugin = plugin; @@ -36,16 +39,24 @@ public void debug(String message) { } }; RedisHandler handler = redisHandler; - - listenerThread = new Thread(() -> { - if (plugin.isEnabled()) { - RedisListener listener = handler.createEnvelopeListener( - plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin_" + plugin.getBungeeSettings().getServer(), - (ch, envelope) -> messageHandler.onMessage(envelope)); - handler.loadListener(listener); + CountDownLatch ready = new CountDownLatch(1); + subscriptionReady = ready; + RedisListener listener = new RedisListener(handler, + plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin_" + plugin.getBungeeSettings().getServer(), + (ch, payload) -> { + try { + messageHandler.onMessage( + com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.decode(payload)); + } catch (Exception e) { + plugin.debug("Redis decode failed: " + e.getMessage()); + } + }) { + @Override + public void onSubscribe(String channel, int subscribedChannels) { + ready.countDown(); } - }, "VotingPlugin-Redis-Backend"); - listenerThread.start(); + }; + handler.loadListener(listener); } @Override @@ -73,6 +84,14 @@ public void validate() { } catch (RuntimeException failure) { throw new IllegalStateException("Redis backend proxy transport connection failed", failure); } + try { + if (subscriptionReady == null || !subscriptionReady.await(3, TimeUnit.SECONDS)) { + throw new IllegalStateException("Redis backend proxy subscription did not become ready"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while waiting for Redis backend proxy subscription", e); + } } @Override @@ -81,9 +100,6 @@ public void close() { redisHandler.close(); redisHandler = null; } - if (listenerThread != null) { - listenerThread.interrupt(); - listenerThread = null; - } + subscriptionReady = null; } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 82057fd926..6adb2c3f6f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -82,6 +82,15 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte installed = true; reload.run(fileName); String applied = readRaw(target, false); + String installedRevision = revision(preview.resolvedContent()); + if (!revision(applied).equals(installedRevision)) { + String concurrentRevision = revision(applied); + reload.run(fileName); + if (!revision(readRaw(target, false)).equals(concurrentRevision)) { + throw new StaleRevisionException(); + } + throw new StaleRevisionException(); + } return new ApplyResult(new Document(fileName, mask(parse(applied)), revision(applied)), preview.changes(), false); } catch (StaleRevisionException stale) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 3aec9593f7..227b74ab6a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -77,7 +77,7 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer } private static ProxyRoutingConfiguration routing(ConfigurationNode configuration) throws IOException { - return new ProxyRoutingConfiguration(configuration.node("SendVotesToAllServers").getBoolean(false), + return new ProxyRoutingConfiguration(configuration.node("SendVotesToAllServers").getBoolean(true), configuration.node("BlockedServers").getList(String.class, List.of())); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java index 9e9c2d0dcc..b436fadd54 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandlerLifecycleTest.java @@ -70,17 +70,27 @@ void releasesSocketListenerBeforeSamePortReplacement() throws Exception { } @Test - void releasesRedisSubscriberBeforeSameTopicReplacement() throws Exception { - RedisBackendProxyTransport handler = new RedisBackendProxyTransport(null); + void keepsRedisSubscriberUntilReplacementIsReady() throws Exception { + BackendProxyHandler handler = new BackendProxyHandler(null); + Field method = BackendProxyHandler.class.getDeclaredField("method"); + method.setAccessible(true); + method.set(handler, BungeeMethod.REDIS); + Field managerField = BackendProxyHandler.class.getDeclaredField("transportManager"); + managerField.setAccessible(true); + BackendProxyTransportManager manager = (BackendProxyTransportManager) managerField.get(handler); + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null); RedisHandler redis = mock(RedisHandler.class); Field field = RedisBackendProxyTransport.class.getDeclaredField("redisHandler"); field.setAccessible(true); - field.set(handler, redis); + field.set(transport, redis); + Field transportField = BackendProxyTransportManager.class.getDeclaredField("transport"); + transportField.setAccessible(true); + transportField.set(manager, transport); - handler.prepareForReplacement(); + handler.prepareForReplacement(BungeeMethod.REDIS); - verify(redis).close(); - assertNull(handler.getRedisHandler()); + verifyNoInteractions(redis); + assertSame(redis, handler.getRedisHandler()); } @Test diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index b8fb938331..242796837d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -122,6 +122,22 @@ class BackendConfigurationServiceTest { assertEquals("Feature: manual\n", Files.readString(config)); } + @Test void successfulReloadDoesNotReportAConcurrentManualEditAsApplied() throws Exception { + Path config = directory.resolve("Config.yml"); + Files.writeString(config, "Feature: before\n"); + AtomicInteger reloads = new AtomicInteger(); + BackendConfigurationService service = new BackendConfigurationService(directory, + (BackendConfigurationService.ApplyAction) fileName -> { + if (reloads.incrementAndGet() == 1) Files.writeString(config, "Feature: manual\n"); + }); + BackendConfigurationService.Document before = service.read("Config.yml"); + + assertThrows(BackendConfigurationService.StaleRevisionException.class, + () -> service.apply("Config.yml", "Feature: proposed\n", before.revision())); + assertEquals(2, reloads.get()); + assertEquals("Feature: manual\n", Files.readString(config)); + } + @Test void quickSetupsProduceReviewableBackendAndVoteSiteChanges() throws Exception { Files.writeString(directory.resolve("BungeeSettings.yml"), "UseBungeecord: false\nServer: PleaseSet\n"); Files.writeString(directory.resolve("VoteSites.yml"), "VoteSites: {}\n"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfigControlTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfigControlTest.java new file mode 100644 index 0000000000..93dcc26bad --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfigControlTest.java @@ -0,0 +1,34 @@ +package com.bencodez.votingplugin.proxy.velocity; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.bencodez.votingplugin.proxy.control.ProxyRoutingConfiguration; + +class VelocityConfigControlTest { + @TempDir Path directory; + + @Test + void omittedSendVotesSettingUsesRuntimeDefaultDuringRevisionCheck() throws Exception { + Path file = directory.resolve("velocity.yml"); + Files.writeString(file, "BlockedServers: []\n"); + VelocityConfig config = new VelocityConfig(file.toFile()); + config.loadControlConfiguration(); + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(config.getSendVotesToAllServers(), + config.getBlockedServers()); + + assertTrue(config.getSendVotesToAllServers()); + config.persistControlProxyRouting(false, List.of("lobby"), current.revision()); + config.loadControlConfiguration(); + + assertFalse(config.getSendVotesToAllServers()); + assertTrue(config.getBlockedServers().contains("lobby")); + } +} From 36f2800aa3ed3a884ffb2f67313bc4f36beec775 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 19:59:31 -0600 Subject: [PATCH 48/73] Keep Redis subscription startup asynchronous --- .../transport/RedisBackendProxyTransport.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index 89d50e2005..d440004e6e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -21,6 +21,7 @@ public class RedisBackendProxyTransport implements BackendProxyTransport { @Getter private RedisHandler redisHandler; private CountDownLatch subscriptionReady; + private Thread listenerThread; public RedisBackendProxyTransport(VotingPluginMain plugin) { this.plugin = plugin; @@ -56,7 +57,9 @@ public void onSubscribe(String channel, int subscribedChannels) { ready.countDown(); } }; - handler.loadListener(listener); + listenerThread = new Thread(() -> handler.loadListener(listener), "VotingPlugin-Redis-Backend"); + listenerThread.setDaemon(true); + listenerThread.start(); } @Override @@ -96,10 +99,20 @@ public void validate() { @Override public void close() { + Thread thread = listenerThread; if (redisHandler != null) { redisHandler.close(); redisHandler = null; } + if (thread != null) { + thread.interrupt(); + try { + thread.join(TimeUnit.SECONDS.toMillis(3)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + listenerThread = null; subscriptionReady = null; } } From 109a38c521760ba17d1f29dee3f02534806bdcc4 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 20:09:39 -0600 Subject: [PATCH 49/73] Harden Control recovery and proxy reloads --- .../proxy/VotingPluginProxyConfig.java | 4 ++ .../proxy/bungee/BungeeConfig.java | 27 ++++++++++-- .../proxy/control/HostedControlManager.java | 43 ++++++++++++++++++- .../ProxyRoutingConfigurationService.java | 12 ++++++ .../proxy/velocity/VelocityConfig.java | 23 +++++++++- .../control/HostedControlManagerTest.java | 34 +++++++++++++++ .../ProxyRoutingConfigurationServiceTest.java | 23 ++++++++++ .../velocity/VelocityConfigControlTest.java | 21 +++++++++ 8 files changed, 179 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java index 4327673078..aab7d3bf90 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java @@ -27,6 +27,10 @@ default void rollbackControlProxyRouting() throws IOException { throw new UnsupportedOperationException("Control configuration rollback is unavailable"); } + /** Verifies that the exact Control-installed routing snapshot is still on disk. */ + default void verifyControlProxyRoutingInstalled() throws IOException { + } + @SuppressWarnings("serial") final class StaleControlRevisionException extends IOException { public StaleControlRevisionException() { super("Control proxy routing revision is stale"); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index 62673dfa3a..b63a35211b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -2,15 +2,20 @@ import java.io.File; import java.io.IOException; -import java.io.InputStream; +import java.io.InputStream; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; -import java.util.Map; +import java.util.Map; +import java.util.Set; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.control.ProxyRoutingConfiguration; @@ -562,14 +567,19 @@ private static ProxyRoutingConfiguration routing(Configuration configuration) { public synchronized void rollbackControlProxyRouting() throws IOException { Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); - if (!Files.isRegularFile(backup)) throw new IOException("Control backup is unavailable"); + if (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Control backup is unavailable or unsafe"); + } if (controlInstalledSnapshot == null || !java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); try { - Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); + try (SeekableByteChannel source = Files.newByteChannel(backup, + Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { + Files.copy(Channels.newInputStream(source), stage, StandardCopyOption.REPLACE_EXISTING); + } if (!java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } @@ -581,6 +591,15 @@ public synchronized void rollbackControlProxyRouting() throws IOException { data = ConfigurationProvider.getProvider(YamlConfiguration.class).load(target.toFile()); } + @Override + public synchronized void verifyControlProxyRoutingInstalled() throws IOException { + Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); + if (controlInstalledSnapshot == null + || !java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + } + private static void atomicReplace(Path source, Path target) throws IOException { try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 4eaa0dd842..59913f306c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -68,6 +68,7 @@ public final class HostedControlManager implements AutoCloseable { private volatile String quarantinedSha256; private volatile String trustedRollbackSha256; private volatile String rollbackCandidateSha256; + private volatile boolean rollbackCandidateReachedHealth; private volatile boolean rollbackPending; private HostedControlManager(Settings settings, Consumer logger) { @@ -125,8 +126,11 @@ void runOnce() { clearManagedProcess(retained); } if (rollbackPending && Files.isRegularFile(settings.previousFile(), LinkOption.NOFOLLOW_LINKS)) { - if (rollbackAndStartPrevious()) return; - throw new IOException("The previous Control release did not become healthy"); + if (rollbackCandidateReachedHealth) { + if (rollbackAndStartPrevious()) return; + throw new IOException("The previous Control release did not become healthy"); + } + restoreIncompleteActivation(); } recoverPersistedQuarantineState(); boolean updated = prepareArtifact(); @@ -139,10 +143,15 @@ void runOnce() { } LaunchedProcess launched = launch(settings.jarFile()); ManagedProcess process = launched.process(); + if (updated) { + persistHealthCheckingState(trustedRollbackSha256, rollbackCandidateSha256); + rollbackCandidateReachedHealth = true; + } if (awaitHealthy(launched)) { failures = 0; clearPersistedRollbackState(); rollbackCandidateSha256 = null; + rollbackCandidateReachedHealth = false; rollbackPending = false; status = Status.RUNNING; logger.accept("WebUI is available at " + settings.endpoint()); @@ -175,6 +184,7 @@ private boolean rollbackAndStartPrevious() throws IOException, InterruptedExcept rollbackPending = false; quarantinedSha256 = failedCandidateSha256; rollbackCandidateSha256 = null; + rollbackCandidateReachedHealth = false; LaunchedProcess previousLaunch = launch(settings.jarFile()); ManagedProcess previous = previousLaunch.process(); if (awaitHealthy(previousLaunch)) { @@ -252,9 +262,14 @@ private void rollback() throws IOException { } private void persistRollbackState(String previousSha256) throws IOException { + Files.deleteIfExists(settings.healthCheckingFile()); persistDigestState(settings.rollbackPendingFile(), previousSha256, settings.sha256(), "rollback"); } + private void persistHealthCheckingState(String previousSha256, String candidateSha256) throws IOException { + persistDigestState(settings.healthCheckingFile(), previousSha256, candidateSha256, "health-checking"); + } + private void persistQuarantineState(String previousSha256, String candidateSha256) throws IOException { persistDigestState(settings.quarantineFile(), previousSha256, candidateSha256, "quarantine"); } @@ -298,6 +313,7 @@ private void recoverPersistedRollbackState() throws IOException { if (previousSha256.equals(activeSha256)) { clearPersistedRollbackState(); rollbackCandidateSha256 = null; + rollbackCandidateReachedHealth = false; rollbackPending = false; return; } @@ -310,9 +326,27 @@ private void recoverPersistedRollbackState() throws IOException { } trustedRollbackSha256 = previousSha256; rollbackCandidateSha256 = candidateSha256; + String[] healthChecking = readDigestState(settings.healthCheckingFile(), "health-checking"); + if (healthChecking != null && (!previousSha256.equals(healthChecking[0]) + || !candidateSha256.equals(healthChecking[1]))) { + throw new IOException("Control health-checking state does not match pending rollback state"); + } + rollbackCandidateReachedHealth = healthChecking != null; rollbackPending = true; } + private void restoreIncompleteActivation() throws IOException { + if (Files.exists(settings.jarFile(), LinkOption.NOFOLLOW_LINKS)) { + atomicMove(settings.jarFile(), settings.failedFile(), true); + } + atomicMove(settings.previousFile(), settings.jarFile(), false); + clearPersistedRollbackState(); + trustedRollbackSha256 = null; + rollbackCandidateSha256 = null; + rollbackCandidateReachedHealth = false; + rollbackPending = false; + } + private void recoverPersistedQuarantineState() throws IOException { String[] lines = readDigestState(settings.quarantineFile(), "quarantine"); if (lines == null) return; @@ -354,6 +388,7 @@ private String[] readDigestState(Path marker, String name) throws IOException { private void clearPersistedRollbackState() throws IOException { Files.deleteIfExists(settings.rollbackPendingFile()); + Files.deleteIfExists(settings.healthCheckingFile()); } private void clearPersistedQuarantineState() throws IOException { @@ -680,6 +715,10 @@ Path quarantineFile() { return jarFile.resolveSibling(jarFile.getFileName() + ".quarantined"); } + Path healthCheckingFile() { + return jarFile.resolveSibling(jarFile.getFileName() + ".health-checking"); + } + URI endpoint() { try { return new URI("http", null, host, port, "/", null, null); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java index e9345ca475..6f6f4b4ce1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java @@ -23,6 +23,9 @@ public ProxyRoutingConfigurationService(VotingPluginProxy proxy) { } @Override public void rollback() throws IOException { proxy.getConfig().rollbackControlProxyRouting(); } @Override public void reload() throws Exception { proxy.reloadControlConfiguration(); } + @Override public void verifyInstalled() throws IOException { + proxy.getConfig().verifyControlProxyRoutingInstalled(); + } }); } @@ -55,6 +58,14 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t } try { platform.reload(); + try { + platform.verifyInstalled(); + } catch (com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException stale) { + platform.reload(); + throw new StaleRevisionException(); + } + } catch (StaleRevisionException stale) { + throw stale; } catch (Exception e) { boolean rolledBack = false; try { @@ -74,6 +85,7 @@ interface Platform { void persist(ProxyRoutingConfiguration proposal, String expectedRevision) throws IOException; void rollback() throws IOException; void reload() throws Exception; + default void verifyInstalled() throws IOException { } } @SuppressWarnings("serial") diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 227b74ab6a..6af8729619 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -2,15 +2,20 @@ import java.io.File; import java.io.IOException; +import java.nio.channels.Channels; +import java.nio.channels.SeekableByteChannel; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.checkerframework.checker.nullness.qual.NonNull; @@ -85,14 +90,19 @@ private static ProxyRoutingConfiguration routing(ConfigurationNode configuration public synchronized void rollbackControlProxyRouting() throws IOException { Path target = configurationFile.toPath(); Path backup = target.resolveSibling(target.getFileName() + ".control-backup"); - if (!Files.isRegularFile(backup)) throw new IOException("Control backup is unavailable"); + if (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Control backup is unavailable or unsafe"); + } if (controlInstalledSnapshot == null || !java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } Path stage = Files.createTempFile(target.getParent(), target.getFileName().toString(), ".control-rollback"); try { - Files.copy(backup, stage, StandardCopyOption.REPLACE_EXISTING); + try (SeekableByteChannel source = Files.newByteChannel(backup, + Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { + Files.copy(Channels.newInputStream(source), stage, StandardCopyOption.REPLACE_EXISTING); + } if (!java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } @@ -104,6 +114,15 @@ public synchronized void rollbackControlProxyRouting() throws IOException { loadControlConfiguration(); } + @Override + public synchronized void verifyControlProxyRoutingInstalled() throws IOException { + Path target = configurationFile.toPath(); + if (controlInstalledSnapshot == null + || !java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + } + /** Loads the active file without the superclass's empty-config fallback. */ public synchronized void loadControlConfiguration() throws IOException { ConfigurationNode loaded = YamlConfigurationLoader.builder().path(configurationFile.toPath()).build().load(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 8064e98194..1919d1c33d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -454,6 +454,40 @@ void quarantinedUpdateDigestSurvivesManagerRestart() throws Exception { restartedManager.close(); } + @Test + void incompleteActivationIsRetriedWithoutQuarantiningCandidate() throws Exception { + byte[] previous = "previous-release".getBytes(StandardCharsets.UTF_8); + byte[] update = "candidate-release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control.jar"); + Files.write(jar, previous); + HostedControlManager.Settings settings = settings(jar, root.resolve("data"), true, true, + digest(update), 1); + AtomicInteger downloads = new AtomicInteger(); + AtomicInteger launches = new AtomicInteger(); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager manager = new HostedControlManager(settings, executor, + (source, target, maximum, timeout) -> { + downloads.incrementAndGet(); + Files.write(target, update); + }, (launchSettings, artifact, launchId) -> { + if (launches.getAndIncrement() == 0) throw new IOException("simulated pre-launch exit"); + return new FakeProcess(); + }, (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, message -> { }); + + manager.runOnce(); + assertEquals(HostedControlManager.Status.FAILED, manager.status()); + assertFalse(Files.exists(settings.healthCheckingFile())); + manager.runOnce(); + + assertEquals(HostedControlManager.Status.RUNNING, manager.status()); + assertEquals(2, downloads.get()); + assertEquals(2, launches.get()); + assertEquals("candidate-release", Files.readString(jar)); + assertFalse(Files.exists(settings.quarantineFile())); + manager.close(); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java index b130474189..9ef084bbcb 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java @@ -70,6 +70,20 @@ void failedReloadPreservesAConcurrentEditDetectedByTheAdapter() { assertTrue(platform.current.blockedServers().contains("manual-edit")); } + @Test + void successfulReloadReconcilesAndRejectsAConcurrentEdit() { + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + FakePlatform platform = new FakePlatform(); + platform.current = current; + platform.manualEditDuringSuccessfulReload = true; + + assertThrows(ProxyRoutingConfigurationService.StaleRevisionException.class, + () -> new ProxyRoutingConfigurationService(platform).apply( + new ProxyRoutingConfiguration(true, List.of()), current.revision())); + assertTrue(platform.reloadCalls == 2); + assertTrue(platform.current.blockedServers().contains("manual-edit")); + } + private static final class FakePlatform implements ProxyRoutingConfigurationService.Platform { private ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); private Set configuredServers = Set.of(); @@ -77,6 +91,7 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ private boolean rolledBack; private boolean staleDuringPersist; private boolean manualEditDuringFailedReload; + private boolean manualEditDuringSuccessfulReload; private ProxyRoutingConfiguration installed; private int reloadCalls; @@ -93,11 +108,19 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ } @Override public void reload() throws Exception { reloadCalls++; + if (manualEditDuringSuccessfulReload && reloadCalls == 1) { + current = new ProxyRoutingConfiguration(false, List.of("manual-edit")); + } if (manualEditDuringFailedReload && reloadCalls == 1) { current = new ProxyRoutingConfiguration(false, List.of("manual-edit")); throw new IOException("invalid reloaded configuration"); } if (failFirstReload && reloadCalls == 1) throw new IOException("invalid reloaded configuration"); } + @Override public void verifyInstalled() throws IOException { + if (!current.equals(installed)) { + throw new com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException(); + } + } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfigControlTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfigControlTest.java index 93dcc26bad..13660a25ab 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfigControlTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfigControlTest.java @@ -1,6 +1,7 @@ package com.bencodez.votingplugin.proxy.velocity; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.nio.file.Files; @@ -31,4 +32,24 @@ void omittedSendVotesSettingUsesRuntimeDefaultDuringRevisionCheck() throws Excep assertFalse(config.getSendVotesToAllServers()); assertTrue(config.getBlockedServers().contains("lobby")); } + + @Test + void rollbackRejectsASymbolicBackupSidecar() throws Exception { + Path file = directory.resolve("velocity.yml"); + Files.writeString(file, "SendVotesToAllServers: true\nBlockedServers: []\n"); + VelocityConfig config = new VelocityConfig(file.toFile()); + config.loadControlConfiguration(); + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(config.getSendVotesToAllServers(), + config.getBlockedServers()); + config.persistControlProxyRouting(false, List.of("lobby"), current.revision()); + Path backup = file.resolveSibling(file.getFileName() + ".control-backup"); + Path external = directory.resolve("external.yml"); + Files.writeString(external, "external: preserved\n"); + Files.delete(backup); + Files.createSymbolicLink(backup, external); + + assertThrows(java.io.IOException.class, config::rollbackControlProxyRouting); + assertTrue(Files.readString(file).contains("lobby")); + assertTrue(Files.readString(external).contains("preserved")); + } } From 67d1ef64684077b1ea2341ab4d00407e9369d0d2 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 20:19:12 -0600 Subject: [PATCH 50/73] Close hosted and connector handoff windows --- .../control/BackendControlConnector.java | 15 ++++++++++----- .../proxy/control/HostedControlManager.java | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 563f42fa9a..abcd395b47 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -225,12 +225,17 @@ private void claimAndExecute() throws Exception { } } JsonObject resultBody = result.json(sessionId); - requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", - resultBody), 200); TaskResult submitted = result; - if (submitted.restartConnector()) { - synchronized (completed) { completed.remove(operationId); } - plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); + try { + requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", + resultBody), 200); + if (submitted.restartConnector()) { + synchronized (completed) { completed.remove(operationId); } + } + } finally { + if (submitted.restartConnector()) { + plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); + } } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 59913f306c..6b8494b7da 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -874,6 +874,7 @@ public ManagedProcess launch(Settings settings, Path artifact, String launchId) environment.put("CONTROL_PORT", Integer.toString(settings.port())); environment.put("CONTROL_DATA_DIR", settings.dataDirectory().toString()); environment.put("CONTROL_LAUNCH_ID", launchId); + environment.put("CONTROL_PARENT_PID", Long.toString(ProcessHandle.current().pid())); return new JdkManagedProcess(builder.start()); } From 78480d1c9ad8303885b354cf68666b0bd1bf6a62 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 20:33:13 -0600 Subject: [PATCH 51/73] Close final Control lifecycle races --- .../proxy/VotingPluginProxyConfig.java | 10 +++++++ .../proxy/bungee/BungeeConfig.java | 13 +++++++++ .../proxy/control/HostedControlManager.java | 11 +++++++- .../ProxyRoutingConfigurationService.java | 23 +++++++++++++++- .../proxy/velocity/VelocityConfig.java | 12 +++++++++ .../control/HostedControlManagerTest.java | 27 +++++++++++++++++++ .../ProxyRoutingConfigurationServiceTest.java | 27 +++++++++++++++++++ 7 files changed, 121 insertions(+), 2 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java index aab7d3bf90..5a02585919 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxyConfig.java @@ -31,6 +31,16 @@ default void rollbackControlProxyRouting() throws IOException { default void verifyControlProxyRoutingInstalled() throws IOException { } + /** Captures the exact proxy configuration file that the next strict reload should consume. */ + default byte[] captureControlProxyRoutingSnapshot() throws IOException { + throw new UnsupportedOperationException("Control configuration snapshot verification is unavailable"); + } + + /** Verifies that the proxy configuration file still equals a previously captured snapshot. */ + default void verifyControlProxyRoutingSnapshot(byte[] snapshot) throws IOException { + throw new UnsupportedOperationException("Control configuration snapshot verification is unavailable"); + } + @SuppressWarnings("serial") final class StaleControlRevisionException extends IOException { public StaleControlRevisionException() { super("Control proxy routing revision is stale"); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index b63a35211b..94d031b35f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -600,6 +600,19 @@ public synchronized void verifyControlProxyRoutingInstalled() throws IOException } } + @Override + public synchronized byte[] captureControlProxyRoutingSnapshot() throws IOException { + return Files.readAllBytes(new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath()); + } + + @Override + public synchronized void verifyControlProxyRoutingSnapshot(byte[] snapshot) throws IOException { + Path target = new File(bungee.getDataFolder(), "bungeeconfig.yml").toPath(); + if (!java.util.Arrays.equals(snapshot, Files.readAllBytes(target))) { + throw new StaleControlRevisionException(); + } + } + private static void atomicReplace(Path source, Path target) throws IOException { try { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 6b8494b7da..0f8e8fb9aa 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -144,7 +144,16 @@ void runOnce() { LaunchedProcess launched = launch(settings.jarFile()); ManagedProcess process = launched.process(); if (updated) { - persistHealthCheckingState(trustedRollbackSha256, rollbackCandidateSha256); + try { + persistHealthCheckingState(trustedRollbackSha256, rollbackCandidateSha256); + } catch (IOException publicationFailure) { + try { + stopProcess(process, true); + } catch (RuntimeException stopFailure) { + publicationFailure.addSuppressed(stopFailure); + } + throw publicationFailure; + } rollbackCandidateReachedHealth = true; } if (awaitHealthy(launched)) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java index 6f6f4b4ce1..5ac9d270c0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java @@ -26,6 +26,12 @@ public ProxyRoutingConfigurationService(VotingPluginProxy proxy) { @Override public void verifyInstalled() throws IOException { proxy.getConfig().verifyControlProxyRoutingInstalled(); } + @Override public byte[] captureInstalledSnapshot() throws IOException { + return proxy.getConfig().captureControlProxyRoutingSnapshot(); + } + @Override public void verifyInstalledSnapshot(byte[] snapshot) throws IOException { + proxy.getConfig().verifyControlProxyRoutingSnapshot(snapshot); + } }); } @@ -61,7 +67,7 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t try { platform.verifyInstalled(); } catch (com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException stale) { - platform.reload(); + reconcileConcurrentEdit(); throw new StaleRevisionException(); } } catch (StaleRevisionException stale) { @@ -79,6 +85,19 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t } } + private void reconcileConcurrentEdit() throws Exception { + for (int attempt = 0; attempt < 3; attempt++) { + byte[] snapshot = platform.captureInstalledSnapshot(); + platform.reload(); + try { + platform.verifyInstalledSnapshot(snapshot); + return; + } catch (com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException stale) { + if (attempt == 2) throw new StaleRevisionException(); + } + } + } + interface Platform { ProxyRoutingConfiguration read(); Set configuredServers(); @@ -86,6 +105,8 @@ interface Platform { void rollback() throws IOException; void reload() throws Exception; default void verifyInstalled() throws IOException { } + default byte[] captureInstalledSnapshot() throws IOException { return new byte[0]; } + default void verifyInstalledSnapshot(byte[] snapshot) throws IOException { } } @SuppressWarnings("serial") diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 6af8729619..b857ff6292 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -123,6 +123,18 @@ public synchronized void verifyControlProxyRoutingInstalled() throws IOException } } + @Override + public synchronized byte[] captureControlProxyRoutingSnapshot() throws IOException { + return Files.readAllBytes(configurationFile.toPath()); + } + + @Override + public synchronized void verifyControlProxyRoutingSnapshot(byte[] snapshot) throws IOException { + if (!java.util.Arrays.equals(snapshot, Files.readAllBytes(configurationFile.toPath()))) { + throw new StaleControlRevisionException(); + } + } + /** Loads the active file without the superclass's empty-config fallback. */ public synchronized void loadControlConfiguration() throws IOException { ConfigurationNode loaded = YamlConfigurationLoader.builder().path(configurationFile.toPath()).build().load(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java index 1919d1c33d..0e8a8b02cf 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/HostedControlManagerTest.java @@ -488,6 +488,33 @@ void incompleteActivationIsRetriedWithoutQuarantiningCandidate() throws Exceptio manager.close(); } + @Test + void launchStatePublicationFailureStopsTheLaunchedChild() throws Exception { + byte[] previous = "previous-release".getBytes(StandardCharsets.UTF_8); + byte[] update = "candidate-release".getBytes(StandardCharsets.UTF_8); + Path root = directory.toAbsolutePath().normalize(); + Path jar = root.resolve("control.jar"); + Files.write(jar, previous); + HostedControlManager.Settings settings = settings(jar, root.resolve("data"), true, true, + digest(update), 1); + FakeProcess process = new FakeProcess(); + ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor(); + HostedControlManager manager = new HostedControlManager(settings, executor, + (source, target, maximum, timeout) -> Files.write(target, update), + (launchSettings, artifact, launchId) -> { + Files.createDirectory(settings.healthCheckingFile()); + Files.writeString(settings.healthCheckingFile().resolve("blocker"), "x"); + return process; + }, (endpoint, timeout, launchId) -> true, millis -> { }, System::nanoTime, message -> { }); + + manager.runOnce(); + + assertEquals(HostedControlManager.Status.FAILED, manager.status()); + assertTrue(process.destroyed); + assertFalse(process.isAlive()); + manager.close(); + } + private HostedControlManager.Settings settings(Path jar, Path data, boolean autoDownload, boolean autoUpdate, String sha256, int startupTimeout) { Path root = directory.toAbsolutePath().normalize(); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java index 9ef084bbcb..ef250979ac 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Set; @@ -84,6 +85,20 @@ void successfulReloadReconcilesAndRejectsAConcurrentEdit() { assertTrue(platform.current.blockedServers().contains("manual-edit")); } + @Test + void reconciliationReloadIsRecheckedAndRepeatedAfterAnotherConcurrentEdit() { + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + FakePlatform platform = new FakePlatform(); + platform.current = current; + platform.manualEditsDuringSuccessfulReloads = 2; + + assertThrows(ProxyRoutingConfigurationService.StaleRevisionException.class, + () -> new ProxyRoutingConfigurationService(platform).apply( + new ProxyRoutingConfiguration(true, List.of()), current.revision())); + assertTrue(platform.reloadCalls == 3); + assertTrue(platform.current.blockedServers().contains("manual-edit-2")); + } + private static final class FakePlatform implements ProxyRoutingConfigurationService.Platform { private ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); private Set configuredServers = Set.of(); @@ -92,6 +107,7 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ private boolean staleDuringPersist; private boolean manualEditDuringFailedReload; private boolean manualEditDuringSuccessfulReload; + private int manualEditsDuringSuccessfulReloads; private ProxyRoutingConfiguration installed; private int reloadCalls; @@ -111,6 +127,9 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ if (manualEditDuringSuccessfulReload && reloadCalls == 1) { current = new ProxyRoutingConfiguration(false, List.of("manual-edit")); } + if (reloadCalls <= manualEditsDuringSuccessfulReloads) { + current = new ProxyRoutingConfiguration(false, List.of("manual-edit-" + reloadCalls)); + } if (manualEditDuringFailedReload && reloadCalls == 1) { current = new ProxyRoutingConfiguration(false, List.of("manual-edit")); throw new IOException("invalid reloaded configuration"); @@ -122,5 +141,13 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ throw new com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException(); } } + @Override public byte[] captureInstalledSnapshot() { + return current.revision().getBytes(StandardCharsets.US_ASCII); + } + @Override public void verifyInstalledSnapshot(byte[] snapshot) throws IOException { + if (!java.util.Arrays.equals(snapshot, current.revision().getBytes(StandardCharsets.US_ASCII))) { + throw new com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException(); + } + } } } From 39b67b74f19c4540098fd7d7a8f50ad873123ae8 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 20:40:54 -0600 Subject: [PATCH 52/73] Finish backend Control shutdown safely --- .../bencodez/votingplugin/VotingPluginMain.java | 10 ++++++++-- .../control/BackendConfigurationService.java | 15 ++++++++++----- .../BackendConfigurationServiceTest.java | 17 +++++++++++++++++ 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 0444184200..020f2799ba 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -898,8 +898,14 @@ public void onFinish() { public void onUnLoad() { BackendControlConnector connector = backendControlConnector; if (connector != null) { - connector.close(); - if (backendControlConnector == connector) backendControlConnector = null; + try { + connector.close(); + } catch (RuntimeException e) { + getLogger().warning("[Control] Bukkit connector did not stop cleanly; normal plugin cleanup will continue"); + debug(e); + } finally { + if (backendControlConnector == connector) backendControlConnector = null; + } } if (getBackendProxyHandler() != null) { try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 6adb2c3f6f..8e2df21e15 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -84,11 +84,7 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte String applied = readRaw(target, false); String installedRevision = revision(preview.resolvedContent()); if (!revision(applied).equals(installedRevision)) { - String concurrentRevision = revision(applied); - reload.run(fileName); - if (!revision(readRaw(target, false)).equals(concurrentRevision)) { - throw new StaleRevisionException(); - } + reconcileConcurrentEdit(fileName, target); throw new StaleRevisionException(); } return new ApplyResult(new Document(fileName, mask(parse(applied)), revision(applied)), @@ -117,6 +113,15 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte } } + private void reconcileConcurrentEdit(String fileName, Path target) throws Exception { + for (int attempt = 0; attempt < 3; attempt++) { + String snapshotRevision = revision(readRaw(target, false)); + reload.run(fileName); + if (revision(readRaw(target, false)).equals(snapshotRevision)) return; + } + throw new StaleRevisionException(); + } + private static void rejectSymbolicBackup(Path backup) throws IOException { if (Files.isSymbolicLink(backup)) throw new IOException("Symbolic configuration backups are not allowed"); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index 242796837d..b0cf6d2ae1 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -138,6 +138,23 @@ class BackendConfigurationServiceTest { assertEquals("Feature: manual\n", Files.readString(config)); } + @Test void reconciliationReloadRetriesWhenAnotherManualEditRacesIt() throws Exception { + Path config = directory.resolve("Config.yml"); + Files.writeString(config, "Feature: before\n"); + AtomicInteger reloads = new AtomicInteger(); + BackendConfigurationService service = new BackendConfigurationService(directory, + (BackendConfigurationService.ApplyAction) fileName -> { + int reload = reloads.incrementAndGet(); + if (reload <= 2) Files.writeString(config, "Feature: manual-" + reload + "\n"); + }); + BackendConfigurationService.Document before = service.read("Config.yml"); + + assertThrows(BackendConfigurationService.StaleRevisionException.class, + () -> service.apply("Config.yml", "Feature: proposed\n", before.revision())); + assertEquals(3, reloads.get()); + assertEquals("Feature: manual-2\n", Files.readString(config)); + } + @Test void quickSetupsProduceReviewableBackendAndVoteSiteChanges() throws Exception { Files.writeString(directory.resolve("BungeeSettings.yml"), "UseBungeecord: false\nServer: PleaseSet\n"); Files.writeString(directory.resolve("VoteSites.yml"), "VoteSites: {}\n"); From 3986db5c87a96a5a19da11f84cbf114be3272bc5 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 20:57:34 -0600 Subject: [PATCH 53/73] Harden Control handoff and final shutdown --- .../votingplugin/VotingPluginMain.java | 18 +++++++++--- .../backendproxy/BackendProxyHandler.java | 2 +- .../cache/ProcessedVoteCache.java | 22 ++++++++++++++ .../BackendProxyTransportManager.java | 9 +++++- .../transport/RedisBackendProxyTransport.java | 18 ++++++++++-- .../control/BackendControlConnector.java | 21 +++++++++++--- .../votingplugin/proxy/VotingPluginProxy.java | 29 ++++++++++++++----- .../votingplugin/proxy/VotingPluginWire.java | 7 +++++ .../proxy/VotingPluginProxyLifecycleTest.java | 23 +++++++++++++++ .../cache/ProcessedVoteCacheTest.java | 10 +++++++ 10 files changed, 138 insertions(+), 21 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 020f2799ba..677b52251e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -757,11 +757,21 @@ private void startBackendControlConnector() { /** Refreshes the optional Control connector after its Config.yml settings were applied. */ public synchronized void restartBackendControlConnector() { BackendControlConnector connector = backendControlConnector; - if (connector != null) { - connector.close(); - if (backendControlConnector == connector) backendControlConnector = null; + BackendControlConnector replacement; + try { + replacement = BackendControlConnector.create(this, connector); + } catch (Exception e) { + if (connector != null && connector.hasPendingResults()) { + getLogger().warning("[Control] Keeping the previous connector until its applied result is acknowledged"); + return; + } + replacement = null; + getLogger().warning("[Control] Bukkit configuration connector was not restarted: " + e.getMessage()); } - startBackendControlConnector(); + if (replacement == null && connector != null && connector.hasPendingResults()) return; + if (connector != null) connector.close(); + if (backendControlConnector == connector) backendControlConnector = replacement; + if (replacement != null) replacement.start(); } /** Recreates proxy transports after Control applies BungeeSettings.yml. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 966b989c3a..279ca02ca9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -54,7 +54,7 @@ public BackendProxyHandler(VotingPluginMain plugin) { public BackendProxyHandler(VotingPluginMain plugin, ProcessedVoteCache processedVoteCache) { this.plugin = plugin; this.processedVoteCache = java.util.Objects.requireNonNull(processedVoteCache, "processedVoteCache"); - transportManager = new BackendProxyTransportManager(plugin); + transportManager = new BackendProxyTransportManager(plugin, processedVoteCache); globalDataSync = new BackendGlobalDataSync(plugin, this::sendEnvelope); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index 0fae384443..b325b5b9e5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -1,6 +1,9 @@ package com.bencodez.votingplugin.backendproxy.cache; import java.util.UUID; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -12,10 +15,12 @@ public class ProcessedVoteCache { private static final long DEFAULT_TTL_MILLIS = TimeUnit.MINUTES.toMillis(30); + private static final int MAX_REDIS_DELIVERIES = 4096; @Getter private final ConcurrentHashMap processedVotes = new ConcurrentHashMap<>(); private final long ttlMillis; + private final LinkedHashMap processedRedisDeliveries = new LinkedHashMap<>(); public ProcessedVoteCache() { this(DEFAULT_TTL_MILLIS); @@ -54,6 +59,23 @@ public boolean reserve(UUID voteId) { } } + /** Deduplicates one Redis envelope across overlapping subscribers during a validated handoff. */ + public synchronized boolean reserveRedisDelivery(String deliveryId) { + if (deliveryId == null || !deliveryId.matches("[0-9a-fA-F-]{36}")) return true; + long now = System.currentTimeMillis(); + processedRedisDeliveries.entrySet().removeIf(entry -> entry.getValue() <= now); + Long current = processedRedisDeliveries.get(deliveryId); + if (current != null && current > now) return false; + processedRedisDeliveries.put(deliveryId, now + ttlMillis); + while (processedRedisDeliveries.size() > MAX_REDIS_DELIVERIES) { + Iterator> oldest = processedRedisDeliveries.entrySet().iterator(); + if (!oldest.hasNext()) break; + oldest.next(); + oldest.remove(); + } + return true; + } + private void cleanup(long now) { processedVotes.entrySet().removeIf(entry -> entry.getValue() <= now); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index 3fd9edd3ec..f0ffc90176 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -8,6 +8,7 @@ import com.bencodez.simpleapi.servercomm.sockets.ClientHandler; import com.bencodez.simpleapi.servercomm.sockets.SocketHandler; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; import com.bencodez.votingplugin.proxy.BungeeMethod; /** @@ -16,10 +17,16 @@ public class BackendProxyTransportManager { private final VotingPluginMain plugin; + private final ProcessedVoteCache processedVoteCache; private BackendProxyTransport transport; public BackendProxyTransportManager(VotingPluginMain plugin) { + this(plugin, new ProcessedVoteCache()); + } + + public BackendProxyTransportManager(VotingPluginMain plugin, ProcessedVoteCache processedVoteCache) { this.plugin = plugin; + this.processedVoteCache = processedVoteCache; } public void start(BungeeMethod method, GlobalMessageHandler messageHandler) { @@ -35,7 +42,7 @@ public void start(BungeeMethod method, GlobalMessageHandler messageHandler) { transport = new SocketBackendProxyTransport(plugin); break; case REDIS: - transport = new RedisBackendProxyTransport(plugin); + transport = new RedisBackendProxyTransport(plugin, processedVoteCache); break; case MQTT: transport = new MqttBackendProxyTransport(plugin); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index d440004e6e..ecfe63fbc8 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -8,6 +8,8 @@ import com.bencodez.simpleapi.servercomm.redis.RedisHandler; import com.bencodez.simpleapi.servercomm.redis.RedisListener; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; +import com.bencodez.votingplugin.proxy.VotingPluginWire; import redis.clients.jedis.DefaultJedisClientConfig; import redis.clients.jedis.HostAndPort; @@ -18,13 +20,19 @@ public class RedisBackendProxyTransport implements BackendProxyTransport { private final VotingPluginMain plugin; + private final ProcessedVoteCache processedVoteCache; @Getter private RedisHandler redisHandler; private CountDownLatch subscriptionReady; private Thread listenerThread; public RedisBackendProxyTransport(VotingPluginMain plugin) { + this(plugin, new ProcessedVoteCache()); + } + + public RedisBackendProxyTransport(VotingPluginMain plugin, ProcessedVoteCache processedVoteCache) { this.plugin = plugin; + this.processedVoteCache = processedVoteCache; } @Override @@ -46,8 +54,11 @@ public void debug(String message) { plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin_" + plugin.getBungeeSettings().getServer(), (ch, payload) -> { try { - messageHandler.onMessage( - com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.decode(payload)); + JsonEnvelope envelope = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.decode(payload); + if (processedVoteCache.reserveRedisDelivery( + envelope.getFields().get(VotingPluginWire.K_REDIS_DELIVERY_ID))) { + messageHandler.onMessage(envelope); + } } catch (Exception e) { plugin.debug("Redis decode failed: " + e.getMessage()); } @@ -65,7 +76,8 @@ public void onSubscribe(String channel, int subscribedChannels) { @Override public void send(JsonEnvelope envelope) { if (redisHandler != null) { - redisHandler.publishEnvelope(plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin", envelope); + redisHandler.publishEnvelope(plugin.getBungeeSettings().getRedisPrefix() + "VotingPlugin", + VotingPluginWire.withRedisDeliveryId(envelope)); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index abcd395b47..ae1f439bda 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -98,6 +98,11 @@ private void reloadConfiguration(String fileName) throws Exception { } public static BackendControlConnector create(VotingPluginMain plugin) throws IOException { + return create(plugin, null); + } + + public static BackendControlConnector create(VotingPluginMain plugin, BackendControlConnector predecessor) + throws IOException { ConfigurationSection control = plugin.getConfigFile().getData().getConfigurationSection("Control.Backend"); if (control == null || !control.getBoolean("Enabled", false)) return null; String nodeId = control.getString("NodeId", "").trim(); @@ -116,7 +121,17 @@ public static BackendControlConnector create(VotingPluginMain plugin) throws IOE bounded(control.getInt("HeartbeatSeconds", 30), 10, 300, "HeartbeatSeconds"), bounded(control.getInt("ConnectTimeoutMillis", 3000), 500, 30000, "ConnectTimeoutMillis"), bounded(control.getInt("RequestTimeoutMillis", 10000), 500, 30000, "RequestTimeoutMillis")); - return new BackendControlConnector(plugin, settings, credential); + BackendControlConnector connector = new BackendControlConnector(plugin, settings, credential); + if (predecessor != null) { + synchronized (predecessor.completed) { + connector.completed.putAll(predecessor.completed); + } + } + return connector; + } + + public boolean hasPendingResults() { + synchronized (completed) { return !completed.isEmpty(); } } public void start() { schedule(0); } @@ -229,9 +244,7 @@ private void claimAndExecute() throws Exception { try { requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", resultBody), 200); - if (submitted.restartConnector()) { - synchronized (completed) { completed.remove(operationId); } - } + synchronized (completed) { completed.remove(operationId); } } finally { if (submitted.restartConnector()) { plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index b12a4f0685..c7f2e17cf6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1608,17 +1608,29 @@ private void startControlServicesLocked() { private void stopControlServicesLocked(boolean waitForHosted) { ControlConnector connector = controlConnector; if (connector != null) { - connector.close(); - if (controlConnector == connector) controlConnector = null; + try { + connector.close(); + if (controlConnector == connector) controlConnector = null; + } catch (RuntimeException failure) { + if (waitForHosted) throw failure; + if (controlConnector == connector) controlConnector = null; + logSevere("[Control] connector did not stop cleanly; proxy cleanup will continue"); + } } HostedControlManager manager = hostedControlManager; if (manager != null) { - if (waitForHosted) { - manager.closeAndWait(); - } else { - manager.close(); + try { + if (waitForHosted) { + manager.closeAndWait(); + } else { + manager.close(); + } + if (hostedControlManager == manager) hostedControlManager = null; + } catch (RuntimeException failure) { + if (waitForHosted) throw failure; + if (hostedControlManager == manager) hostedControlManager = null; + logSevere("[Control Host] manager did not stop cleanly; proxy cleanup will continue"); } - if (hostedControlManager == manager) hostedControlManager = null; } } @@ -2682,7 +2694,8 @@ private boolean sendRedisEnvelopeServer(String server, JsonEnvelope envelope, bo try (Jedis jedis = publisherPool.getResource()) { String channel = getConfig().getRedisPrefix() + "VotingPlugin_" + server; - long subscribers = jedis.publish(channel, JsonEnvelopeCodec.encode(envelope)); + long subscribers = jedis.publish(channel, + JsonEnvelopeCodec.encode(VotingPluginWire.withRedisDeliveryId(envelope))); redisPublisherRetryAfter = 0L; return subscribers > 0; } catch (Exception e) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java index f1837a41c1..ad1449a57e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginWire.java @@ -27,6 +27,7 @@ * reference them directly. */ public final class VotingPluginWire { + public static final String K_REDIS_DELIVERY_ID = "_vpRedisDeliveryId"; private static final int MAX_PRESENCE_SNAPSHOT_JSON_LENGTH = 65536; private VotingPluginWire() { @@ -636,6 +637,12 @@ private static JsonEnvelope.Builder base(String subChannel) { return JsonEnvelope.builder(subChannel).schema(ver); } + /** Adds a per-publication identity so overlapping Redis subscribers can suppress one duplicate delivery. */ + public static JsonEnvelope withRedisDeliveryId(JsonEnvelope envelope) { + if (envelope.getFields().containsKey(K_REDIS_DELIVERY_ID)) return envelope; + return envelope.toBuilder().put(K_REDIS_DELIVERY_ID, UUID.randomUUID().toString()).build(); + } + private static String safe(String s) { return s == null ? "" : s; } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java index df745ba910..32b68ac8eb 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/VotingPluginProxyLifecycleTest.java @@ -4,6 +4,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; import java.lang.reflect.Field; @@ -57,6 +58,28 @@ void retainsHostedManagerWhenBoundedShutdownFails() throws Exception { assertSame(manager, hosted.get(proxy)); } + @Test + void finalStopContinuesToHostedManagerAfterConnectorDrainFailure() throws Exception { + VotingPluginProxyTestImpl proxy = new VotingPluginProxyTestImpl(); + ControlConnector connector = mock(ControlConnector.class); + doThrow(new IllegalStateException("operation still running")).when(connector).close(); + HostedControlManager manager = mock(HostedControlManager.class); + Field control = VotingPluginProxy.class.getDeclaredField("controlConnector"); + control.setAccessible(true); + control.set(proxy, connector); + Field hosted = VotingPluginProxy.class.getDeclaredField("hostedControlManager"); + hosted.setAccessible(true); + hosted.set(proxy, manager); + Method stop = VotingPluginProxy.class.getDeclaredMethod("stopControlServices", boolean.class); + stop.setAccessible(true); + + stop.invoke(proxy, false); + + verify(manager).close(); + assertNull(control.get(proxy)); + assertNull(hosted.get(proxy)); + } + @Test void stopsEveryReplacedSocketClientEvenWhenOneStopFails() { ClientHandler failing = mock(ClientHandler.class); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java index f98fce6761..0873f1f079 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java @@ -38,4 +38,14 @@ public void expiredVoteIdCanBeReservedAgain() throws Exception { assertTrue(cache.reserve(voteId)); assertEquals(1, cache.getProcessedVotes().size()); } + + @Test + public void redisDeliveryIsAcceptedOnceAcrossSharedHandlers() { + ProcessedVoteCache cache = new ProcessedVoteCache(); + String deliveryId = UUID.randomUUID().toString(); + assertTrue(cache.reserveRedisDelivery(deliveryId)); + assertFalse(cache.reserveRedisDelivery(deliveryId)); + assertTrue(cache.reserveRedisDelivery(UUID.randomUUID().toString())); + assertTrue(cache.reserveRedisDelivery(null)); + } } From 8ca35f2d052727091f24346f5b60811c3310b0ac Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 21:08:46 -0600 Subject: [PATCH 54/73] Separate Control and proxy reload shutdown --- .../votingplugin/proxy/VotingPluginProxy.java | 21 ++++++++++++++++--- .../proxy/bungee/VotingPluginBungee.java | 11 ++++++++-- .../proxy/velocity/VotingPluginVelocity.java | 10 +++++++-- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index c7f2e17cf6..242c19fc5e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -2351,10 +2351,25 @@ public void onDisable() { /** Full runtime replacement waits for hosted workers; final proxy stop remains non-blocking. */ public void onDisable(boolean waitForHosted) { + if (waitForHosted) { + prepareForRuntimeReplacement(); + } else { + controlServicesGeneration.incrementAndGet(); + controlLifecycleExecutor.shutdownNow(); + stopControlServices(false); + } + completeRuntimeReplacementShutdown(); + } + + /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ + public void prepareForRuntimeReplacement() { controlServicesGeneration.incrementAndGet(); - if (waitForHosted) controlLifecycleExecutor.shutdown(); - else controlLifecycleExecutor.shutdownNow(); - stopControlServices(waitForHosted); + controlLifecycleExecutor.shutdown(); + stopControlServices(true); + } + + /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ + public void completeRuntimeReplacementShutdown() { getVoteCacheHandler().saveVoteCache(); if (getProxyMysqlMessenger() != null) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index 784cccd587..a75359e34c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -393,10 +393,10 @@ public void reloadPlugin(boolean loadMysql) { } catch (Exception ignored) { } - // Tear down old runtime (this WILL shutdown MySQL because proxy owns it) + // Stop Control first; replacement must not overlap a retained hosted child/connector. try { if (votingPluginProxy != null) { - votingPluginProxy.onDisable(true); + votingPluginProxy.prepareForRuntimeReplacement(); } } catch (Exception shutdownFailure) { getLogger().severe("Reload aborted because hosted Control did not stop safely"); @@ -409,6 +409,13 @@ public void reloadPlugin(boolean loadMysql) { reloading = false; return; } + // Later transport/cache failures must not leave the old runtime partially disabled. + try { + if (votingPluginProxy != null) votingPluginProxy.completeRuntimeReplacementShutdown(); + } catch (Exception cleanupFailure) { + getLogger().severe("Old proxy runtime cleanup was incomplete; replacement will continue"); + cleanupFailure.printStackTrace(); + } // Recreate runtime votingPluginProxy = createProxyRuntime(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 235b50a5af..7e8af5eb55 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -478,10 +478,10 @@ public void reloadAllInternal(boolean loadMysql) { } catch (Exception ignored) { } - // Shutdown old runtime completely (this stops old MySQL pool) + // Stop Control first; replacement must not overlap a retained hosted child/connector. try { if (votingPluginProxy != null) { - votingPluginProxy.onDisable(true); + votingPluginProxy.prepareForRuntimeReplacement(); } } catch (Exception shutdownFailure) { logger.error("Reload aborted because hosted Control did not stop safely", shutdownFailure); @@ -493,6 +493,12 @@ public void reloadAllInternal(boolean loadMysql) { reloading = false; return; } + // Later transport/cache failures must not leave the old runtime partially disabled. + try { + if (votingPluginProxy != null) votingPluginProxy.completeRuntimeReplacementShutdown(); + } catch (Exception cleanupFailure) { + logger.error("Old proxy runtime cleanup was incomplete; replacement will continue", cleanupFailure); + } // Recreate runtime votingPluginProxy = createProxyRuntime(); From 373354ae15587406610f7698d3a27f73429f968b Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 21:18:19 -0600 Subject: [PATCH 55/73] Gate legacy Redis handler handoff --- .../votingplugin/VotingPluginMain.java | 1 + .../backendproxy/BackendProxyHandler.java | 7 +++ .../cache/ProcessedVoteCache.java | 48 +++++++++++++++++ .../BackendProxyTransportManager.java | 15 ++++++ .../transport/RedisBackendProxyTransport.java | 53 +++++++++++++++++-- .../cache/ProcessedVoteCacheTest.java | 16 ++++++ 6 files changed, 137 insertions(+), 3 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 677b52251e..df0016ecbd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -788,6 +788,7 @@ public synchronized void restartBackendProxyHandler() { try { replacement.load(); replacement.validateTransport(); + if (previous != null) previous.completeRedisHandoff(replacement); } catch (RuntimeException failure) { replacement.close(); throw failure; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java index 279ca02ca9..8395c38b4c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/BackendProxyHandler.java @@ -116,6 +116,13 @@ public void validateTransport() { transportManager.validate(); } + /** Completes the no-loss/no-duplicate same-Redis subscriber handoff after validation. */ + public void completeRedisHandoff(BackendProxyHandler replacement) { + if (method != BungeeMethod.REDIS || replacement.method != BungeeMethod.REDIS) return; + transportManager.closeRedisForHandoff(); + replacement.transportManager.activateRedisAfterHandoff(); + } + public void playerOnline(String playerName, String uuid) { if (presenceManager != null) { presenceManager.playerOnline(playerName, uuid); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index b325b5b9e5..e91e2914f0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -21,6 +21,9 @@ public class ProcessedVoteCache { private final ConcurrentHashMap processedVotes = new ConcurrentHashMap<>(); private final long ttlMillis; private final LinkedHashMap processedRedisDeliveries = new LinkedHashMap<>(); + private final LinkedHashMap legacyRedisDeliveries = new LinkedHashMap<>(); + private Object activeRedisSubscriber; + private Object standbyRedisSubscriber; public ProcessedVoteCache() { this(DEFAULT_TTL_MILLIS); @@ -76,6 +79,51 @@ public synchronized boolean reserveRedisDelivery(String deliveryId) { return true; } + public synchronized void registerRedisSubscriber(Object subscriber) { + if (activeRedisSubscriber == null) { + activeRedisSubscriber = subscriber; + } else if (activeRedisSubscriber != subscriber) { + standbyRedisSubscriber = subscriber; + legacyRedisDeliveries.clear(); + } + } + + /** Returns true only for the active subscriber and counts its legacy delivery during overlap. */ + public synchronized boolean reserveLegacyRedisDelivery(Object subscriber, String signature) { + if (activeRedisSubscriber != subscriber) return false; + if (standbyRedisSubscriber != null) legacyRedisDeliveries.merge(signature, 1, Integer::sum); + return true; + } + + public synchronized void activateRedisSubscriber(Object subscriber) { + if (standbyRedisSubscriber != subscriber) { + throw new IllegalStateException("Redis replacement subscriber is not registered"); + } + activeRedisSubscriber = subscriber; + standbyRedisSubscriber = null; + } + + /** Consumes one matching delivery processed by the previous active subscriber. */ + public synchronized boolean consumeLegacyRedisDelivery(String signature) { + Integer count = legacyRedisDeliveries.get(signature); + if (count == null) return false; + if (count <= 1) legacyRedisDeliveries.remove(signature); + else legacyRedisDeliveries.put(signature, count - 1); + return true; + } + + public synchronized void finishRedisHandoff() { + legacyRedisDeliveries.clear(); + } + + public synchronized void unregisterRedisSubscriber(Object subscriber) { + if (standbyRedisSubscriber == subscriber) { + standbyRedisSubscriber = null; + legacyRedisDeliveries.clear(); + } + if (activeRedisSubscriber == subscriber) activeRedisSubscriber = null; + } + private void cleanup(long now) { processedVotes.entrySet().removeIf(entry -> entry.getValue() <= now); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java index f0ffc90176..4d7b208721 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/BackendProxyTransportManager.java @@ -78,6 +78,21 @@ public void prepareForReplacement() { } } + public void closeRedisForHandoff() { + if (!(transport instanceof RedisBackendProxyTransport)) { + throw new IllegalStateException("Redis backend proxy transport is unavailable"); + } + ((RedisBackendProxyTransport) transport).closeForHandoff(); + transport = null; + } + + public void activateRedisAfterHandoff() { + if (!(transport instanceof RedisBackendProxyTransport)) { + throw new IllegalStateException("Redis replacement transport is unavailable"); + } + ((RedisBackendProxyTransport) transport).activateAfterHandoff(); + } + public ClientHandler getClientHandler() { return transport instanceof SocketBackendProxyTransport ? ((SocketBackendProxyTransport) transport).getClientHandler() : null; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index ecfe63fbc8..ab47e38ccb 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -2,6 +2,8 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; +import java.util.ArrayList; +import java.util.List; import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; @@ -25,6 +27,10 @@ public class RedisBackendProxyTransport implements BackendProxyTransport { private RedisHandler redisHandler; private CountDownLatch subscriptionReady; private Thread listenerThread; + private final Object subscriberIdentity = new Object(); + private final Object legacyLifecycle = new Object(); + private final List bufferedLegacyDeliveries = new ArrayList<>(); + private GlobalMessageHandler messageHandler; public RedisBackendProxyTransport(VotingPluginMain plugin) { this(plugin, new ProcessedVoteCache()); @@ -37,6 +43,8 @@ public RedisBackendProxyTransport(VotingPluginMain plugin, ProcessedVoteCache pr @Override public void start(GlobalMessageHandler messageHandler) { + this.messageHandler = messageHandler; + processedVoteCache.registerRedisSubscriber(subscriberIdentity); redisHandler = new RedisHandler(plugin.getBungeeSettings().getRedisHost(), plugin.getBungeeSettings().getRedisPort(), plugin.getBungeeSettings().getRedisUsername(), plugin.getBungeeSettings().getRedisPassword(), plugin.getBungeeSettings().getRedisdbindex()) { @@ -55,9 +63,11 @@ public void debug(String message) { (ch, payload) -> { try { JsonEnvelope envelope = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.decode(payload); - if (processedVoteCache.reserveRedisDelivery( - envelope.getFields().get(VotingPluginWire.K_REDIS_DELIVERY_ID))) { - messageHandler.onMessage(envelope); + String deliveryId = envelope.getFields().get(VotingPluginWire.K_REDIS_DELIVERY_ID); + if (deliveryId != null) { + if (processedVoteCache.reserveRedisDelivery(deliveryId)) messageHandler.onMessage(envelope); + } else { + dispatchLegacy(envelope); } } catch (Exception e) { plugin.debug("Redis decode failed: " + e.getMessage()); @@ -73,6 +83,35 @@ public void onSubscribe(String channel, int subscribedChannels) { listenerThread.start(); } + private void dispatchLegacy(JsonEnvelope envelope) { + String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); + synchronized (legacyLifecycle) { + if (processedVoteCache.reserveLegacyRedisDelivery(subscriberIdentity, signature)) { + messageHandler.onMessage(envelope); + } else { + bufferedLegacyDeliveries.add(envelope); + } + } + } + + /** Promotes a validated standby after the previous listener has completely stopped. */ + public void activateAfterHandoff() { + synchronized (legacyLifecycle) { + processedVoteCache.activateRedisSubscriber(subscriberIdentity); + for (JsonEnvelope envelope : bufferedLegacyDeliveries) { + String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); + if (!processedVoteCache.consumeLegacyRedisDelivery(signature)) messageHandler.onMessage(envelope); + } + bufferedLegacyDeliveries.clear(); + processedVoteCache.finishRedisHandoff(); + } + } + + /** Stops the old listener while retaining its overlap accounting for standby promotion. */ + public void closeForHandoff() { + closeListener(false); + } + @Override public void send(JsonEnvelope envelope) { if (redisHandler != null) { @@ -111,6 +150,10 @@ public void validate() { @Override public void close() { + closeListener(true); + } + + private void closeListener(boolean unregister) { Thread thread = listenerThread; if (redisHandler != null) { redisHandler.close(); @@ -122,9 +165,13 @@ public void close() { thread.join(TimeUnit.SECONDS.toMillis(3)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while stopping Redis backend subscription", e); } + if (thread.isAlive()) throw new IllegalStateException("Redis backend subscription did not stop"); } listenerThread = null; subscriptionReady = null; + messageHandler = null; + if (unregister) processedVoteCache.unregisterRedisSubscriber(subscriberIdentity); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java index 0873f1f079..fe54a6184d 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java @@ -48,4 +48,20 @@ public void redisDeliveryIsAcceptedOnceAcrossSharedHandlers() { assertTrue(cache.reserveRedisDelivery(UUID.randomUUID().toString())); assertTrue(cache.reserveRedisDelivery(null)); } + + @Test + public void legacyRedisHandoffMatchesOldAndBufferedDeliveryCounts() { + ProcessedVoteCache cache = new ProcessedVoteCache(); + Object previous = new Object(); + Object replacement = new Object(); + cache.registerRedisSubscriber(previous); + cache.registerRedisSubscriber(replacement); + + assertTrue(cache.reserveLegacyRedisDelivery(previous, "same-envelope")); + assertFalse(cache.reserveLegacyRedisDelivery(replacement, "same-envelope")); + cache.activateRedisSubscriber(replacement); + assertTrue(cache.consumeLegacyRedisDelivery("same-envelope")); + assertFalse(cache.consumeLegacyRedisDelivery("same-envelope")); + cache.finishRedisHandoff(); + } } From d330ff88d236642123cacec3a46584066e91de19 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 21:26:53 -0600 Subject: [PATCH 56/73] Carry proxy Control results across reload --- .../votingplugin/proxy/VotingPluginProxy.java | 10 ++++++---- .../proxy/control/ControlConnector.java | 18 ++++++++++++++++-- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 242c19fc5e..4558f32cb7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1552,8 +1552,9 @@ public void onReceive(JsonEnvelope message) { private void startControlServices() { synchronized (controlLifecycleLock) { + ControlConnector predecessor = controlConnector; stopControlServicesLocked(true); - startControlServicesLocked(); + startControlServicesLocked(predecessor); } } @@ -1565,8 +1566,9 @@ private void restartControlServicesAsync() { try { synchronized (controlLifecycleLock) { if (!enabled || generation != controlServicesGeneration.get()) return; + ControlConnector predecessor = controlConnector; stopControlServicesLocked(true); - startControlServicesLocked(); + startControlServicesLocked(predecessor); } } catch (RuntimeException failure) { if (generation == controlServicesGeneration.get()) { @@ -1585,7 +1587,7 @@ private void stopControlServices(boolean waitForHosted) { } } - private void startControlServicesLocked() { + private void startControlServicesLocked(ControlConnector predecessor) { if (getConfig().getControlHostedEnabled()) { try { hostedControlManager = HostedControlManager.create(this); @@ -1597,7 +1599,7 @@ private void startControlServicesLocked() { } if (!getConfig().getControlEnabled()) return; try { - controlConnector = ControlConnector.create(this); + controlConnector = ControlConnector.create(this, predecessor); if (controlConnector != null) controlConnector.start(); } catch (IOException | IllegalArgumentException e) { controlConnector = null; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 074080fe63..3cba1353fc 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -95,6 +95,11 @@ public ControlConnector(Settings settings, ScheduledExecutorService scheduler, T /** Creates the production connector without reading a credential when the feature is disabled. */ public static ControlConnector create(VotingPluginProxy proxy) throws IOException { + return create(proxy, null); + } + + /** Creates a replacement while preserving results that Control has not acknowledged. */ + public static ControlConnector create(VotingPluginProxy proxy, ControlConnector predecessor) throws IOException { VotingPluginProxyConfig config = proxy.getConfig(); if (!config.getControlEnabled()) { return null; @@ -132,9 +137,15 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio backends.sort(Comparator.comparing(ObservedBackend::backendId)); return List.copyOf(backends); }; - return new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, + ControlConnector connector = new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, message -> proxy.log("[Control] " + message), UUID.randomUUID(), () -> ThreadLocalRandom.current().nextLong(), new ProxyRoutingConfigurationService(proxy)); + if (predecessor != null) { + synchronized (predecessor.completedTasks) { + connector.completedTasks.putAll(predecessor.completedTasks); + } + } + return connector; } public void start() { @@ -365,7 +376,10 @@ private CompletableFuture handleClaimResponse(Response response) { } CompletableFuture submitted = transport.send(resultRequest(operationId, result)); activeRequest = submitted; - return submitted.thenAccept(ControlConnector::requireSuccess); + return submitted.thenAccept(resultResponse -> { + requireSuccess(resultResponse); + synchronized (completedTasks) { completedTasks.remove(operationId); } + }); } private TaskResult executeTask(JsonObject task) { From 1a4c2d0995f3ede456899e99f3b3d06fa282bca6 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 21:35:43 -0600 Subject: [PATCH 57/73] Complete proxy runtime result handoff --- .../votingplugin/proxy/VotingPluginProxy.java | 91 +++++++++++-------- .../proxy/bungee/VotingPluginBungee.java | 7 +- .../proxy/control/ControlConnector.java | 23 +++-- .../proxy/velocity/VotingPluginVelocity.java | 7 +- 4 files changed, 81 insertions(+), 47 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 4558f32cb7..5f0e0c0ed1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -167,6 +167,7 @@ public abstract class VotingPluginProxy { private final Map pendingPresenceHandoffs = new HashMap<>(); private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); private volatile ControlConnector controlConnector; + private volatile ControlConnector.PendingResults inheritedControlResults; private volatile HostedControlManager hostedControlManager; private final Object controlLifecycleLock = new Object(); private final AtomicLong controlServicesGeneration = new AtomicLong(); @@ -1599,7 +1600,10 @@ private void startControlServicesLocked(ControlConnector predecessor) { } if (!getConfig().getControlEnabled()) return; try { - controlConnector = ControlConnector.create(this, predecessor); + ControlConnector.PendingResults pending = predecessor == null + ? inheritedControlResults : predecessor.pendingResults(); + controlConnector = ControlConnector.createWithPendingResults(this, pending); + inheritedControlResults = null; if (controlConnector != null) controlConnector.start(); } catch (IOException | IllegalArgumentException e) { controlConnector = null; @@ -2367,53 +2371,66 @@ public void onDisable(boolean waitForHosted) { public void prepareForRuntimeReplacement() { controlServicesGeneration.incrementAndGet(); controlLifecycleExecutor.shutdown(); + ControlConnector connector = controlConnector; + if (connector != null) inheritedControlResults = connector.pendingResults(); stopControlServices(true); } + /** Transfers unacknowledged apply results into a newly created platform runtime. */ + public void inheritPendingControlResults(VotingPluginProxy previous) { + ControlConnector connector = previous == null ? null : previous.controlConnector; + if (connector != null) inheritedControlResults = connector.pendingResults(); + else if (previous != null) inheritedControlResults = previous.inheritedControlResults; + } + /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ public void completeRuntimeReplacementShutdown() { - getVoteCacheHandler().saveVoteCache(); - - if (getProxyMysqlMessenger() != null) { - getProxyMysqlMessenger().shutdown(); - } - - if (getProxyMySQL() != null) { - getProxyMySQL().shutdown(); - } - if (multiProxyHandler != null) { - multiProxyHandler.close(); - } - - if (socketHandler != null) { - socketHandler.closeConnection(); - } - closeSocketClients(); - - if (redisHandler != null) { - redisHandler.close(); - } - if (redisPublisherPool != null) { - redisPublisherPool.close(); - redisPublisherPool = null; - } - if (mqttHandler != null) { + runCleanup("vote cache", () -> getVoteCacheHandler().saveVoteCache()); + runCleanup("proxy MySQL messenger", () -> { + if (getProxyMysqlMessenger() != null) getProxyMysqlMessenger().shutdown(); + }); + runCleanup("proxy MySQL", () -> { + if (getProxyMySQL() != null) getProxyMySQL().shutdown(); + }); + runCleanup("multi-proxy handler", () -> { + if (multiProxyHandler != null) multiProxyHandler.close(); + }); + runCleanup("socket listener", () -> { + if (socketHandler != null) socketHandler.closeConnection(); + }); + runCleanup("socket clients", this::closeSocketClients); + runCleanup("Redis subscriber", () -> { + if (redisHandler != null) redisHandler.close(); + }); + runCleanup("Redis publisher", () -> { + JedisPool pool = redisPublisherPool; try { - mqttHandler.disconnect(); - } catch (Exception e) { - logSevere("Unable to disconnect the MQTT proxy transport"); + if (pool != null) pool.close(); + } finally { + if (redisPublisherPool == pool) redisPublisherPool = null; } - } - - bungeeTimeChecker.shutdown(); + }); + runCleanup("MQTT transport", () -> { + if (mqttHandler != null) mqttHandler.disconnect(); + }); + runCleanup("time checker", () -> bungeeTimeChecker.shutdown()); + runCleanup("global data", () -> { + if (getGlobalDataHandler() != null) getGlobalDataHandler().shutdown(); + }); + enabled = false; + } - if (getGlobalDataHandler() != null) { - getGlobalDataHandler().shutdown(); + private void runCleanup(String service, CleanupAction cleanup) { + try { + cleanup.run(); + } catch (Exception failure) { + logSevere("Unable to stop " + service + "; remaining proxy cleanup will continue"); } - - enabled = false; } + @FunctionalInterface + private interface CleanupAction { void run() throws Exception; } + public void onPluginMessageReceived(DataInputStream in) { runAsync(() -> { try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index a75359e34c..fad3f5b6fd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -417,8 +417,11 @@ public void reloadPlugin(boolean loadMysql) { cleanupFailure.printStackTrace(); } - // Recreate runtime - votingPluginProxy = createProxyRuntime(); + // Recreate runtime and carry results captured before the old connector stopped. + VotingPluginProxy previousRuntime = votingPluginProxy; + VotingPluginProxy replacementRuntime = createProxyRuntime(); + replacementRuntime.inheritPendingControlResults(previousRuntime); + votingPluginProxy = replacementRuntime; // Initialize MySQL BEFORE calling proxy.load(...) try { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 3cba1353fc..9d1a74ba36 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -95,11 +95,16 @@ public ControlConnector(Settings settings, ScheduledExecutorService scheduler, T /** Creates the production connector without reading a credential when the feature is disabled. */ public static ControlConnector create(VotingPluginProxy proxy) throws IOException { - return create(proxy, null); + return createWithPendingResults(proxy, null); } /** Creates a replacement while preserving results that Control has not acknowledged. */ public static ControlConnector create(VotingPluginProxy proxy, ControlConnector predecessor) throws IOException { + return createWithPendingResults(proxy, predecessor == null ? null : predecessor.pendingResults()); + } + + public static ControlConnector createWithPendingResults(VotingPluginProxy proxy, PendingResults pending) + throws IOException { VotingPluginProxyConfig config = proxy.getConfig(); if (!config.getControlEnabled()) { return null; @@ -140,14 +145,14 @@ public static ControlConnector create(VotingPluginProxy proxy, ControlConnector ControlConnector connector = new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, message -> proxy.log("[Control] " + message), UUID.randomUUID(), () -> ThreadLocalRandom.current().nextLong(), new ProxyRoutingConfigurationService(proxy)); - if (predecessor != null) { - synchronized (predecessor.completedTasks) { - connector.completedTasks.putAll(predecessor.completedTasks); - } - } + if (pending != null) connector.completedTasks.putAll(pending.tasks); return connector; } + public PendingResults pendingResults() { + synchronized (completedTasks) { return new PendingResults(new LinkedHashMap<>(completedTasks)); } + } + public void start() { if (closed) { return; @@ -615,6 +620,12 @@ private static TaskResult failure(String code, String message) { } } + /** Opaque immutable handoff state used when a platform recreates its entire proxy runtime. */ + public static final class PendingResults { + private final Map tasks; + private PendingResults(Map tasks) { this.tasks = Map.copyOf(tasks); } + } + public interface Transport extends AutoCloseable { CompletableFuture send(Request request); @Override default void close() { } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 7e8af5eb55..487de7c161 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -500,8 +500,11 @@ public void reloadAllInternal(boolean loadMysql) { logger.error("Old proxy runtime cleanup was incomplete; replacement will continue", cleanupFailure); } - // Recreate runtime - votingPluginProxy = createProxyRuntime(); + // Recreate runtime and carry results captured before the old connector stopped. + VotingPluginProxy previousRuntime = votingPluginProxy; + VotingPluginProxy replacementRuntime = createProxyRuntime(); + replacementRuntime.inheritPendingControlResults(previousRuntime); + votingPluginProxy = replacementRuntime; // Init MySQL BEFORE calling proxy.load(...) try { From cc7606f4a5c23683493a082c086b43146eb61051 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 26 Aug 2026 21:42:27 -0600 Subject: [PATCH 58/73] Snapshot Control results after drain --- .../com/bencodez/votingplugin/proxy/VotingPluginProxy.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 5f0e0c0ed1..dfdf757f74 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1616,6 +1616,7 @@ private void stopControlServicesLocked(boolean waitForHosted) { if (connector != null) { try { connector.close(); + if (waitForHosted) inheritedControlResults = connector.pendingResults(); if (controlConnector == connector) controlConnector = null; } catch (RuntimeException failure) { if (waitForHosted) throw failure; @@ -2371,8 +2372,6 @@ public void onDisable(boolean waitForHosted) { public void prepareForRuntimeReplacement() { controlServicesGeneration.incrementAndGet(); controlLifecycleExecutor.shutdown(); - ControlConnector connector = controlConnector; - if (connector != null) inheritedControlResults = connector.pendingResults(); stopControlServices(true); } From caf011e2c5ab711fccb79bc5599af8779f812f3c Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 18:32:33 -0600 Subject: [PATCH 59/73] Honor Redis SSL during replacement validation --- .../transport/RedisBackendProxyTransport.java | 27 +++++++++++++------ .../RedisBackendProxyTransportTest.java | 27 +++++++++++++++++++ 2 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index fcc6ac7a32..ce0d56a959 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -5,6 +5,8 @@ import java.util.ArrayList; import java.util.List; +import javax.net.ssl.SSLParameters; + import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; import com.bencodez.simpleapi.servercomm.redis.RedisHandler; @@ -124,15 +126,10 @@ public void send(JsonEnvelope envelope) { @Override public void validate() { if (redisHandler == null) throw new IllegalStateException("Redis backend proxy transport initialization failed"); - DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder() - .database(plugin.getBungeeSettings().getRedisdbindex()) - .connectionTimeoutMillis(2000).socketTimeoutMillis(2000); - String username = plugin.getBungeeSettings().getRedisUsername(); - String password = plugin.getBungeeSettings().getRedisPassword(); - if (username != null && !username.isEmpty()) config.user(username); - if (password != null && !password.isEmpty()) config.password(password); try (Jedis jedis = new Jedis(new HostAndPort(plugin.getBungeeSettings().getRedisHost(), - plugin.getBungeeSettings().getRedisPort()), config.build())) { + plugin.getBungeeSettings().getRedisPort()), buildValidationClientConfig( + plugin.getBungeeSettings().getRedisdbindex(), plugin.getBungeeSettings().getRedisUsername(), + plugin.getBungeeSettings().getRedisPassword(), plugin.getBungeeSettings().isRedisSsl()))) { if (!"PONG".equalsIgnoreCase(jedis.ping())) { throw new IllegalStateException("Redis backend proxy transport did not answer PING"); } @@ -149,6 +146,20 @@ public void validate() { } } + static DefaultJedisClientConfig buildValidationClientConfig(int database, String username, String password, + boolean ssl) { + DefaultJedisClientConfig.Builder config = DefaultJedisClientConfig.builder().database(database).ssl(ssl) + .connectionTimeoutMillis(2000).socketTimeoutMillis(2000); + if (ssl) { + SSLParameters sslParameters = new SSLParameters(); + sslParameters.setEndpointIdentificationAlgorithm("HTTPS"); + config.sslParameters(sslParameters); + } + if (username != null && !username.isEmpty()) config.user(username); + if (password != null && !password.isEmpty()) config.password(password); + return config.build(); + } + @Override public void close() { closeListener(true); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java new file mode 100644 index 0000000000..af46ee44f5 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -0,0 +1,27 @@ +package com.bencodez.votingplugin.backendproxy.transport; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +import redis.clients.jedis.DefaultJedisClientConfig; + +class RedisBackendProxyTransportTest { + + @Test + void validationHonorsTlsAndHostnameVerification() { + DefaultJedisClientConfig config = RedisBackendProxyTransport.buildValidationClientConfig(3, "user", "secret", + true); + + assertTrue(config.isSsl()); + assertEquals(3, config.getDatabase()); + assertEquals("HTTPS", config.getSslParameters().getEndpointIdentificationAlgorithm()); + } + + @Test + void validationKeepsTlsDisabledByDefault() { + assertFalse(RedisBackendProxyTransport.buildValidationClientConfig(0, null, null, false).isSsl()); + } +} From 7c7d9d059f528b9e76e5d344453df6a4630abd1d Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 18:55:39 -0600 Subject: [PATCH 60/73] Acknowledge Control results before connector restart --- .../votingplugin/VotingPluginMain.java | 11 +++--- .../control/BackendControlConnector.java | 38 +++++++++---------- .../BackendControlConnectorProtocolTest.java | 13 +++++++ 3 files changed, 35 insertions(+), 27 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index df0016ecbd..59cc62654d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -757,18 +757,17 @@ private void startBackendControlConnector() { /** Refreshes the optional Control connector after its Config.yml settings were applied. */ public synchronized void restartBackendControlConnector() { BackendControlConnector connector = backendControlConnector; + if (connector != null && connector.hasPendingResults()) { + getLogger().warning("[Control] Keeping the previous connector until its applied result is acknowledged"); + return; + } BackendControlConnector replacement; try { - replacement = BackendControlConnector.create(this, connector); + replacement = BackendControlConnector.create(this); } catch (Exception e) { - if (connector != null && connector.hasPendingResults()) { - getLogger().warning("[Control] Keeping the previous connector until its applied result is acknowledged"); - return; - } replacement = null; getLogger().warning("[Control] Bukkit configuration connector was not restarted: " + e.getMessage()); } - if (replacement == null && connector != null && connector.hasPendingResults()) return; if (connector != null) connector.close(); if (backendControlConnector == connector) backendControlConnector = replacement; if (replacement != null) replacement.start(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index ae1f439bda..bcc520903e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -98,11 +98,6 @@ private void reloadConfiguration(String fileName) throws Exception { } public static BackendControlConnector create(VotingPluginMain plugin) throws IOException { - return create(plugin, null); - } - - public static BackendControlConnector create(VotingPluginMain plugin, BackendControlConnector predecessor) - throws IOException { ConfigurationSection control = plugin.getConfigFile().getData().getConfigurationSection("Control.Backend"); if (control == null || !control.getBoolean("Enabled", false)) return null; String nodeId = control.getString("NodeId", "").trim(); @@ -121,13 +116,7 @@ public static BackendControlConnector create(VotingPluginMain plugin, BackendCon bounded(control.getInt("HeartbeatSeconds", 30), 10, 300, "HeartbeatSeconds"), bounded(control.getInt("ConnectTimeoutMillis", 3000), 500, 30000, "ConnectTimeoutMillis"), bounded(control.getInt("RequestTimeoutMillis", 10000), 500, 30000, "RequestTimeoutMillis")); - BackendControlConnector connector = new BackendControlConnector(plugin, settings, credential); - if (predecessor != null) { - synchronized (predecessor.completed) { - connector.completed.putAll(predecessor.completed); - } - } - return connector; + return new BackendControlConnector(plugin, settings, credential); } public boolean hasPendingResults() { @@ -241,15 +230,20 @@ private void claimAndExecute() throws Exception { } JsonObject resultBody = result.json(sessionId); TaskResult submitted = result; - try { - requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", - resultBody), 200); - synchronized (completed) { completed.remove(operationId); } - } finally { - if (submitted.restartConnector()) { - plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); - } - } + afterResultAcknowledged( + () -> requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + + "/result", resultBody), 200), + () -> { + synchronized (completed) { completed.remove(operationId); } + if (submitted.restartConnector()) { + plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); + } + }); + } + + static void afterResultAcknowledged(ResultSubmission submission, Runnable acknowledged) throws Exception { + submission.submit(); + acknowledged.run(); } private TaskResult execute(JsonObject task) { @@ -423,6 +417,8 @@ static void awaitShutdown(ScheduledExecutorService executor, CompletableFuture changes, boolean reloaded, boolean rolledBack, diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index 20ca56847d..ebb451d867 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -5,9 +5,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.util.concurrent.CompletableFuture; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.Test; @@ -43,4 +45,15 @@ class BackendControlConnectorProtocolTest { closing.get(2, TimeUnit.SECONDS); assertTrue(executor.isTerminated()); } + + @Test void failedResultAcknowledgementDoesNotTriggerConnectorHandoff() throws Exception { + AtomicBoolean handedOff = new AtomicBoolean(); + assertThrows(IOException.class, () -> BackendControlConnector.afterResultAcknowledged( + () -> { throw new IOException("Control result was not acknowledged"); }, + () -> handedOff.set(true))); + assertFalse(handedOff.get()); + + BackendControlConnector.afterResultAcknowledged(() -> { }, () -> handedOff.set(true)); + assertTrue(handedOff.get()); + } } From b87b0680a3e46031b015aa247fe440d56cbc95e6 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 19:14:44 -0600 Subject: [PATCH 61/73] Keep Control results durable and origin-bound --- .../control/BackendControlConnector.java | 111 +++++++++--- .../control/BackendControlResultStore.java | 159 ++++++++++++++++++ .../votingplugin/proxy/VotingPluginProxy.java | 39 +++-- .../proxy/bungee/VotingPluginBungee.java | 4 +- .../proxy/control/ControlConnector.java | 82 +++++---- .../proxy/velocity/VotingPluginVelocity.java | 4 +- .../BackendControlResultStoreTest.java | 50 ++++++ .../proxy/control/ControlConnectorTest.java | 25 +++ 8 files changed, 400 insertions(+), 74 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index bcc520903e..59f429f0cd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -28,6 +28,8 @@ import org.bukkit.configuration.ConfigurationSection; import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.control.BackendControlResultStore.Route; +import com.bencodez.votingplugin.control.BackendControlResultStore.StoredResult; import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; import com.bencodez.votingplugin.util.ControlCredentialFile; import com.google.gson.JsonArray; @@ -44,13 +46,15 @@ public final class BackendControlConnector implements AutoCloseable { private static final Set CAPABILITIES = Set.of("config.files.v1", "config.quick-setup.v1"); private final VotingPluginMain plugin; + private final Path dataDirectory; private final Settings settings; private final String credential; private final ScheduledExecutorService executor; private final HttpClient http; private final BackendConfigurationService configurations; private final UUID sessionId = UUID.randomUUID(); - private final Map completed = new LinkedHashMap<>(); + private final Map completed = new LinkedHashMap<>(); + private final boolean recovering; private final AtomicBoolean running = new AtomicBoolean(); private final Object operationLifecycle = new Object(); private volatile boolean closed; @@ -62,10 +66,13 @@ public final class BackendControlConnector implements AutoCloseable { private volatile Future activeReload; private volatile CompletableFuture activeOperation; - private BackendControlConnector(VotingPluginMain plugin, Settings settings, String credential) { + private BackendControlConnector(VotingPluginMain plugin, Path dataDirectory, Settings settings, String credential, + boolean recovering) { this.plugin = plugin; + this.dataDirectory = dataDirectory; this.settings = settings; this.credential = credential; + this.recovering = recovering; ThreadFactory factory = runnable -> { Thread thread = new Thread(runnable, "votingplugin-control-backend"); thread.setDaemon(true); @@ -98,6 +105,15 @@ private void reloadConfiguration(String fileName) throws Exception { } public static BackendControlConnector create(VotingPluginMain plugin) throws IOException { + Path root = plugin.getDataFolder().toPath().toAbsolutePath().normalize(); + BackendControlResultStore.State recovered = BackendControlResultStore.load(root); + if (recovered != null) { + Settings settings = Settings.from(recovered.route()); + String credential = ControlCredentialFile.read(root, settings.credentialFile()); + BackendControlConnector connector = new BackendControlConnector(plugin, root, settings, credential, true); + connector.completed.putAll(recovered.results()); + return connector; + } ConfigurationSection control = plugin.getConfigFile().getData().getConfigurationSection("Control.Backend"); if (control == null || !control.getBoolean("Enabled", false)) return null; String nodeId = control.getString("NodeId", "").trim(); @@ -109,14 +125,13 @@ public static BackendControlConnector create(VotingPluginMain plugin) throws IOE || (endpoint.getPath() != null && !endpoint.getPath().isEmpty() && !"/".equals(endpoint.getPath()))) { throw new IllegalArgumentException("Control.Backend.Endpoint must be an HTTP(S) origin"); } - Path root = plugin.getDataFolder().toPath().toAbsolutePath().normalize(); - String credential = ControlCredentialFile.read(root, - control.getString("CredentialFile", "control-credential.txt")); - Settings settings = new Settings(nodeId, endpoint, + String credentialFile = control.getString("CredentialFile", "control-credential.txt"); + String credential = ControlCredentialFile.read(root, credentialFile); + Settings settings = new Settings(nodeId, endpoint, credentialFile, bounded(control.getInt("HeartbeatSeconds", 30), 10, 300, "HeartbeatSeconds"), bounded(control.getInt("ConnectTimeoutMillis", 3000), 500, 30000, "ConnectTimeoutMillis"), bounded(control.getInt("RequestTimeoutMillis", 10000), 500, 30000, "RequestTimeoutMillis")); - return new BackendControlConnector(plugin, settings, credential); + return new BackendControlConnector(plugin, root, settings, credential, false); } public boolean hasPendingResults() { @@ -210,6 +225,7 @@ private JsonObject heartbeat() throws Exception { } private void claimAndExecute() throws Exception { + if (submitCompletedResult()) return; JsonObject body = new JsonObject(); body.addProperty("sessionId", sessionId.toString()); Response response = send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations", body); @@ -217,33 +233,64 @@ private void claimAndExecute() throws Exception { JsonObject task = requireObject(response, 200); if (closed) return; UUID operationId = UUID.fromString(string(task, "operationId")); - TaskResult result; + StoredResult result; synchronized (completed) { result = completed.get(operationId); } if (result == null) { - result = execute(task); + TaskResult executed = execute(task); + result = new StoredResult(executed.json(), executed.restartConnector()); synchronized (completed) { completed.put(operationId, result); - while (completed.size() > 128) completed.remove(completed.keySet().iterator().next()); } + persistCompleted(); } - JsonObject resultBody = result.json(sessionId); - TaskResult submitted = result; + submitCompletedResult(operationId, result); + } + + private boolean submitCompletedResult() throws Exception { + Map.Entry pending; + synchronized (completed) { + pending = completed.entrySet().stream().findFirst().orElse(null); + } + if (pending == null) return false; + persistCompleted(); + submitCompletedResult(pending.getKey(), pending.getValue()); + return true; + } + + private void submitCompletedResult(UUID operationId, StoredResult submitted) throws Exception { + JsonObject resultBody = submitted.result().deepCopy(); + resultBody.addProperty("sessionId", sessionId.toString()); afterResultAcknowledged( () -> requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", resultBody), 200), () -> { synchronized (completed) { completed.remove(operationId); } - if (submitted.restartConnector()) { + try { + persistCompleted(); + } catch (Exception failure) { + synchronized (completed) { completed.put(operationId, submitted); } + throw failure; + } + boolean drained; + synchronized (completed) { drained = completed.isEmpty(); } + if (drained && (submitted.restartConnector() || recovering)) { plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); } }); } - static void afterResultAcknowledged(ResultSubmission submission, Runnable acknowledged) throws Exception { + private void persistCompleted() throws IOException { + Map snapshot; + synchronized (completed) { snapshot = new LinkedHashMap<>(completed); } + BackendControlResultStore.save(dataDirectory, settings.route(), snapshot); + } + + static void afterResultAcknowledged(ResultSubmission submission, ResultAcknowledgement acknowledged) + throws Exception { submission.submit(); - acknowledged.run(); + acknowledged.acknowledge(); } private TaskResult execute(JsonObject task) { @@ -414,18 +461,44 @@ static void awaitShutdown(ScheduledExecutorService executor, CompletableFuture 2048) { + throw new IllegalArgumentException("Control.Backend.CredentialFile is invalid"); + } + bounded(heartbeatSeconds, 10, 300, "HeartbeatSeconds"); + bounded(connectTimeoutMillis, 500, 30000, "ConnectTimeoutMillis"); + bounded(requestTimeoutMillis, 500, 30000, "RequestTimeoutMillis"); + } + + private static Settings from(Route route) { + return new Settings(route.nodeId(), route.endpoint(), route.credentialFile(), route.heartbeatSeconds(), + route.connectTimeoutMillis(), route.requestTimeoutMillis()); + } + + private Route route() { + return new Route(nodeId, endpoint, credentialFile, heartbeatSeconds, connectTimeoutMillis, + requestTimeoutMillis); + } + } private record Response(int status, String body) { } @FunctionalInterface interface ResultSubmission { void submit() throws Exception; } + @FunctionalInterface + interface ResultAcknowledgement { void acknowledge() throws Exception; } private record TaskResult(boolean success, String code, String message, String revision, JsonObject configuration, List changes, boolean reloaded, boolean rolledBack, boolean restartConnector) { - private JsonObject json(UUID sessionId) { + private JsonObject json() { JsonObject body = new JsonObject(); - body.addProperty("sessionId", sessionId.toString()); body.addProperty("success", success); body.addProperty("code", code); body.addProperty("message", message); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java new file mode 100644 index 0000000000..193e096a3a --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java @@ -0,0 +1,159 @@ +package com.bencodez.votingplugin.control; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** Durable result journal used to make locally applied Control operations restart-safe. */ +final class BackendControlResultStore { + private static final int VERSION = 1; + private static final int MAX_BYTES = 1024 * 1024; + private static final int MAX_RESULTS = 128; + private static final String FILE_NAME = ".control-pending-results.json"; + + private BackendControlResultStore() { } + + static State load(Path dataDirectory) throws IOException { + Path target = target(dataDirectory); + if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) return null; + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target) + || Files.size(target) > MAX_BYTES) { + throw new IOException("Control pending-result journal is unsafe or too large"); + } + byte[] bytes; + try (SeekableByteChannel channel = Files.newByteChannel(target, + Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { + ByteBuffer buffer = ByteBuffer.allocate((int) Files.size(target) + 1); + while (channel.read(buffer) >= 0 && buffer.hasRemaining()) { } + if (!buffer.hasRemaining()) throw new IOException("Control pending-result journal is too large"); + buffer.flip(); + bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + } + try { + JsonElement parsed = JsonParser.parseString(new String(bytes, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw invalid(); + JsonObject root = parsed.getAsJsonObject(); + if (integer(root, "version") != VERSION) throw invalid(); + JsonObject routeJson = object(root, "route"); + Route route = new Route(string(routeJson, "nodeId"), URI.create(string(routeJson, "endpoint")), + string(routeJson, "credentialFile"), integer(routeJson, "heartbeatSeconds"), + integer(routeJson, "connectTimeoutMillis"), integer(routeJson, "requestTimeoutMillis")); + JsonArray listed = array(root, "results"); + if (listed.size() == 0 || listed.size() > MAX_RESULTS) throw invalid(); + Map results = new LinkedHashMap<>(); + for (JsonElement element : listed) { + if (!element.isJsonObject()) throw invalid(); + JsonObject item = element.getAsJsonObject(); + UUID operationId = UUID.fromString(string(item, "operationId")); + JsonObject result = object(item, "result").deepCopy(); + if (!item.has("restartConnector") || !item.get("restartConnector").isJsonPrimitive()) throw invalid(); + StoredResult previous = results.put(operationId, + new StoredResult(result, item.get("restartConnector").getAsBoolean())); + if (previous != null) throw invalid(); + } + return new State(route, Map.copyOf(results)); + } catch (RuntimeException e) { + throw new IOException("Control pending-result journal is malformed", e); + } + } + + static void save(Path dataDirectory, Route route, Map results) throws IOException { + Path target = target(dataDirectory); + if (results.isEmpty()) { + if (Files.isSymbolicLink(target)) throw new IOException("Control pending-result journal is unsafe"); + Files.deleteIfExists(target); + return; + } + if (results.size() > MAX_RESULTS) throw new IOException("Too many pending Control results"); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS) + && (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target))) { + throw new IOException("Control pending-result journal is unsafe"); + } + JsonObject root = new JsonObject(); + root.addProperty("version", VERSION); + JsonObject routeJson = new JsonObject(); + routeJson.addProperty("nodeId", route.nodeId()); + routeJson.addProperty("endpoint", route.endpoint().toString()); + routeJson.addProperty("credentialFile", route.credentialFile()); + routeJson.addProperty("heartbeatSeconds", route.heartbeatSeconds()); + routeJson.addProperty("connectTimeoutMillis", route.connectTimeoutMillis()); + routeJson.addProperty("requestTimeoutMillis", route.requestTimeoutMillis()); + root.add("route", routeJson); + JsonArray listed = new JsonArray(); + results.forEach((operationId, result) -> { + JsonObject item = new JsonObject(); + item.addProperty("operationId", operationId.toString()); + item.add("result", result.result().deepCopy()); + item.addProperty("restartConnector", result.restartConnector()); + listed.add(item); + }); + root.add("results", listed); + byte[] bytes = root.toString().getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_BYTES) throw new IOException("Control pending-result journal is too large"); + Files.createDirectories(dataDirectory); + Path staging = Files.createTempFile(dataDirectory, ".control-results-", ".json"); + try { + Files.write(staging, bytes, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + try { + Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(staging); + } + } + + private static Path target(Path dataDirectory) { + return dataDirectory.toAbsolutePath().normalize().resolve(FILE_NAME); + } + + private static JsonObject object(JsonObject object, String name) { + if (!object.has(name) || !object.get(name).isJsonObject()) throw invalid(); + return object.getAsJsonObject(name); + } + + private static JsonArray array(JsonObject object, String name) { + if (!object.has(name) || !object.get(name).isJsonArray()) throw invalid(); + return object.getAsJsonArray(name); + } + + private static String string(JsonObject object, String name) { + if (!object.has(name) || !object.get(name).isJsonPrimitive()) throw invalid(); + String value = object.get(name).getAsString(); + if (value == null || value.isBlank() || value.length() > 2048) throw invalid(); + return value; + } + + private static int integer(JsonObject object, String name) { + if (!object.has(name) || !object.get(name).isJsonPrimitive()) throw invalid(); + return object.get(name).getAsInt(); + } + + private static IllegalArgumentException invalid() { + return new IllegalArgumentException("invalid pending-result journal"); + } + + record Route(String nodeId, URI endpoint, String credentialFile, int heartbeatSeconds, + int connectTimeoutMillis, int requestTimeoutMillis) { } + record StoredResult(JsonObject result, boolean restartConnector) { } + record State(Route route, Map results) { } +} diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index a3d02350db..9245c6474a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -169,7 +169,6 @@ public abstract class VotingPluginProxy { private final Map pendingPresenceHandoffs = new HashMap<>(); private final Set pendingBackendRecoverySnapshots = ConcurrentHashMap.newKeySet(); private volatile ControlConnector controlConnector; - private volatile ControlConnector.PendingResults inheritedControlResults; private volatile HostedControlManager hostedControlManager; private final Object controlLifecycleLock = new Object(); private final AtomicLong controlServicesGeneration = new AtomicLong(); @@ -1557,8 +1556,12 @@ public void onReceive(JsonEnvelope message) { private void startControlServices() { synchronized (controlLifecycleLock) { ControlConnector predecessor = controlConnector; + if (predecessor != null && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { + log("[Control] service restart deferred until the current result is acknowledged"); + return; + } stopControlServicesLocked(true); - startControlServicesLocked(predecessor); + startControlServicesLocked(); } } @@ -1571,8 +1574,13 @@ private void restartControlServicesAsync() { synchronized (controlLifecycleLock) { if (!enabled || generation != controlServicesGeneration.get()) return; ControlConnector predecessor = controlConnector; + if (predecessor != null + && predecessor.deferReplacementUntilSafe(this::restartControlServicesAsync)) { + log("[Control] service restart deferred until the current result is acknowledged"); + return; + } stopControlServicesLocked(true); - startControlServicesLocked(predecessor); + startControlServicesLocked(); } } catch (RuntimeException failure) { if (generation == controlServicesGeneration.get()) { @@ -1591,7 +1599,7 @@ private void stopControlServices(boolean waitForHosted) { } } - private void startControlServicesLocked(ControlConnector predecessor) { + private void startControlServicesLocked() { if (getConfig().getControlHostedEnabled()) { try { hostedControlManager = HostedControlManager.create(this); @@ -1603,10 +1611,7 @@ private void startControlServicesLocked(ControlConnector predecessor) { } if (!getConfig().getControlEnabled()) return; try { - ControlConnector.PendingResults pending = predecessor == null - ? inheritedControlResults : predecessor.pendingResults(); - controlConnector = ControlConnector.createWithPendingResults(this, pending); - inheritedControlResults = null; + controlConnector = ControlConnector.create(this); if (controlConnector != null) controlConnector.start(); } catch (IOException | IllegalArgumentException e) { controlConnector = null; @@ -1619,7 +1624,6 @@ private void stopControlServicesLocked(boolean waitForHosted) { if (connector != null) { try { connector.close(); - if (waitForHosted) inheritedControlResults = connector.pendingResults(); if (controlConnector == connector) controlConnector = null; } catch (RuntimeException failure) { if (waitForHosted) throw failure; @@ -2379,15 +2383,14 @@ public void onDisable(boolean waitForHosted) { /** Fail-closed gate that must complete before a replacement proxy runtime is created. */ public void prepareForRuntimeReplacement() { controlServicesGeneration.incrementAndGet(); - controlLifecycleExecutor.shutdown(); - stopControlServices(true); - } - - /** Transfers unacknowledged apply results into a newly created platform runtime. */ - public void inheritPendingControlResults(VotingPluginProxy previous) { - ControlConnector connector = previous == null ? null : previous.controlConnector; - if (connector != null) inheritedControlResults = connector.pendingResults(); - else if (previous != null) inheritedControlResults = previous.inheritedControlResults; + synchronized (controlLifecycleLock) { + ControlConnector connector = controlConnector; + if (connector != null && !connector.reserveRuntimeReplacement()) { + throw new IllegalStateException("Control result must be acknowledged before proxy runtime replacement"); + } + controlLifecycleExecutor.shutdown(); + stopControlServicesLocked(true); + } } /** Best-effort remainder of runtime teardown after the Control overlap gate has succeeded. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java index fad3f5b6fd..f567245a95 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/VotingPluginBungee.java @@ -417,10 +417,8 @@ public void reloadPlugin(boolean loadMysql) { cleanupFailure.printStackTrace(); } - // Recreate runtime and carry results captured before the old connector stopped. - VotingPluginProxy previousRuntime = votingPluginProxy; + // Recreate the runtime only after the old connector has drained its result. VotingPluginProxy replacementRuntime = createProxyRuntime(); - replacementRuntime.inheritPendingControlResults(previousRuntime); votingPluginProxy = replacementRuntime; // Initialize MySQL BEFORE calling proxy.load(...) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 9d1a74ba36..07fed482a3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -63,7 +63,9 @@ public final class ControlConnector implements AutoCloseable { private final LongSupplier jitterSource; private final ProxyRoutingConfigurationService configurationService; private final Map completedTasks = new LinkedHashMap<>(); + private final Object operationLifecycle = new Object(); private final AtomicBoolean inFlight = new AtomicBoolean(); + private Runnable deferredReplacement; private volatile boolean closed; private volatile boolean registered; private volatile boolean configurationAccepted; @@ -95,16 +97,6 @@ public ControlConnector(Settings settings, ScheduledExecutorService scheduler, T /** Creates the production connector without reading a credential when the feature is disabled. */ public static ControlConnector create(VotingPluginProxy proxy) throws IOException { - return createWithPendingResults(proxy, null); - } - - /** Creates a replacement while preserving results that Control has not acknowledged. */ - public static ControlConnector create(VotingPluginProxy proxy, ControlConnector predecessor) throws IOException { - return createWithPendingResults(proxy, predecessor == null ? null : predecessor.pendingResults()); - } - - public static ControlConnector createWithPendingResults(VotingPluginProxy proxy, PendingResults pending) - throws IOException { VotingPluginProxyConfig config = proxy.getConfig(); if (!config.getControlEnabled()) { return null; @@ -145,14 +137,9 @@ public static ControlConnector createWithPendingResults(VotingPluginProxy proxy, ControlConnector connector = new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, message -> proxy.log("[Control] " + message), UUID.randomUUID(), () -> ThreadLocalRandom.current().nextLong(), new ProxyRoutingConfigurationService(proxy)); - if (pending != null) connector.completedTasks.putAll(pending.tasks); return connector; } - public PendingResults pendingResults() { - synchronized (completedTasks) { return new PendingResults(new LinkedHashMap<>(completedTasks)); } - } - public void start() { if (closed) { return; @@ -165,6 +152,30 @@ public Status status() { return status; } + /** Defers a replacement while a cycle or unacknowledged result still belongs to this connector. */ + public boolean deferReplacementUntilSafe(Runnable replacement) { + Objects.requireNonNull(replacement, "replacement"); + synchronized (operationLifecycle) { + if (inFlight.get() || !completedTasks.isEmpty()) { + deferredReplacement = replacement; + return true; + } + closed = true; + status = Status.STOPPED; + return false; + } + } + + /** Reserves a quiescent connector for a full runtime replacement without losing a claimed result. */ + public boolean reserveRuntimeReplacement() { + synchronized (operationLifecycle) { + if (inFlight.get() || !completedTasks.isEmpty()) return false; + closed = true; + status = Status.STOPPED; + return true; + } + } + private void schedule(long delayMillis) { if (closed) { return; @@ -179,15 +190,15 @@ private void schedule(long delayMillis) { } void cycle() { - if (closed || !inFlight.compareAndSet(false, true)) { - return; + synchronized (operationLifecycle) { + if (closed || !inFlight.compareAndSet(false, true)) return; } Request first = registered ? heartbeatRequest() : registrationRequest(); CompletableFuture primary; try { primary = transport.send(first); } catch (RuntimeException failure) { - inFlight.set(false); + finishCycle(); onFailure(failure); return; } @@ -197,7 +208,7 @@ void cycle() { primary.cancel(true); operationDone.complete(null); if (activeOperation == operationDone) activeOperation = null; - inFlight.set(false); + finishCycle(); return; } activeRequest = primary; @@ -217,7 +228,7 @@ void cycle() { operation.whenComplete((ignored, failure) -> { try { activeRequest = null; - inFlight.set(false); + finishCycle(); if (!closed) { if (failure == null) { onSuccess(); @@ -236,6 +247,18 @@ void cycle() { }); } + private void finishCycle() { + Runnable replacement = null; + synchronized (operationLifecycle) { + inFlight.set(false); + if (completedTasks.isEmpty() && deferredReplacement != null) { + replacement = deferredReplacement; + deferredReplacement = null; + } + } + if (replacement != null) replacement.run(); + } + private void handlePrimaryResponse(Response response, boolean registration) { if (response.statusCode == 404 && !registration) { registered = false; @@ -369,12 +392,12 @@ private CompletableFuture handleClaimResponse(Response response) { JsonObject task = parseObject(response.body); UUID operationId = UUID.fromString(requireString(task, "operationId")); TaskResult result; - synchronized (completedTasks) { + synchronized (operationLifecycle) { result = completedTasks.get(operationId); } if (result == null) { result = executeTask(task); - synchronized (completedTasks) { + synchronized (operationLifecycle) { completedTasks.put(operationId, result); while (completedTasks.size() > 128) completedTasks.remove(completedTasks.keySet().iterator().next()); } @@ -383,7 +406,7 @@ private CompletableFuture handleClaimResponse(Response response) { activeRequest = submitted; return submitted.thenAccept(resultResponse -> { requireSuccess(resultResponse); - synchronized (completedTasks) { completedTasks.remove(operationId); } + synchronized (operationLifecycle) { completedTasks.remove(operationId); } }); } @@ -498,8 +521,11 @@ static String stableBackendId(String serverName) { @Override public void close() { - closed = true; - status = Status.STOPPED; + synchronized (operationLifecycle) { + closed = true; + status = Status.STOPPED; + deferredReplacement = null; + } ScheduledFuture scheduledRequest = scheduled; if (scheduledRequest != null) { scheduledRequest.cancel(false); @@ -620,12 +646,6 @@ private static TaskResult failure(String code, String message) { } } - /** Opaque immutable handoff state used when a platform recreates its entire proxy runtime. */ - public static final class PendingResults { - private final Map tasks; - private PendingResults(Map tasks) { this.tasks = Map.copyOf(tasks); } - } - public interface Transport extends AutoCloseable { CompletableFuture send(Request request); @Override default void close() { } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java index 487de7c161..09d33ed750 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VotingPluginVelocity.java @@ -500,10 +500,8 @@ public void reloadAllInternal(boolean loadMysql) { logger.error("Old proxy runtime cleanup was incomplete; replacement will continue", cleanupFailure); } - // Recreate runtime and carry results captured before the old connector stopped. - VotingPluginProxy previousRuntime = votingPluginProxy; + // Recreate the runtime only after the old connector has drained its result. VotingPluginProxy replacementRuntime = createProxyRuntime(); - replacementRuntime.inheritPendingControlResults(previousRuntime); votingPluginProxy = replacementRuntime; // Init MySQL BEFORE calling proxy.load(...) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java new file mode 100644 index 0000000000..5162f38de6 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java @@ -0,0 +1,50 @@ +package com.bencodez.votingplugin.control; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.bencodez.votingplugin.control.BackendControlResultStore.Route; +import com.bencodez.votingplugin.control.BackendControlResultStore.StoredResult; +import com.google.gson.JsonObject; + +class BackendControlResultStoreTest { + @TempDir Path directory; + + @Test void pendingResultAndOriginalRouteSurviveRestartUntilAcknowledged() throws Exception { + UUID operationId = UUID.fromString("00000000-0000-0000-0000-000000000099"); + Route route = new Route("backend-old", URI.create("https://control.example:8443"), "old-credential.txt", + 30, 3000, 10000); + JsonObject result = new JsonObject(); + result.addProperty("success", true); + result.addProperty("revision", "applied-revision"); + Map pending = new LinkedHashMap<>(); + pending.put(operationId, new StoredResult(result, true)); + + BackendControlResultStore.save(directory, route, pending); + BackendControlResultStore.State recovered = BackendControlResultStore.load(directory); + + assertEquals(route, recovered.route()); + assertEquals("applied-revision", recovered.results().get(operationId).result().get("revision").getAsString()); + BackendControlResultStore.save(directory, route, Map.of()); + assertFalse(Files.exists(directory.resolve(".control-pending-results.json"))); + } + + @Test void symbolicPendingResultJournalIsRejected() throws Exception { + Path external = directory.resolve("external.json"); + Files.writeString(external, "{}"); + Files.createSymbolicLink(directory.resolve(".control-pending-results.json"), external); + + assertThrows(java.io.IOException.class, () -> BackendControlResultStore.load(directory)); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index b93097abf9..6038b67102 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -17,6 +17,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -197,6 +198,28 @@ class ControlConnectorTest { } } + @Test void replacementWaitsForTheOriginatingNodeToAcknowledgeItsResult() { + connector.close(); + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L, + new ProxyRoutingConfigurationService(new NoOpPlatform())); + transport.acceptConfiguration = true; + transport.operationClaim = CompletableFuture.completedFuture(new Response(200, + "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"type\":\"APPLY\",\"expectedRevision\":\"" + current.revision() + "\"," + + "\"configuration\":{\"sendVotesToAllServers\":true,\"blockedServers\":[]}}")); + transport.resultSubmission = new CompletableFuture<>(); + connector.cycle(); + + AtomicBoolean replacementStarted = new AtomicBoolean(); + assertTrue(connector.deferReplacementUntilSafe(() -> replacementStarted.set(true))); + assertFalse(replacementStarted.get()); + + transport.resultSubmission.complete(new Response(200, "{}")); + assertTrue(replacementStarted.get()); + } + @Test void closeBeforeOperationPublicationPreventsTheRequestChainFromStarting() throws Exception { connector.close(); transport.firstSendEntered = new CountDownLatch(1); @@ -253,6 +276,7 @@ private static final class FakeTransport implements Transport { private RuntimeException synchronousFailure; private boolean acceptConfiguration; private CompletableFuture operationClaim; + private CompletableFuture resultSubmission; private CountDownLatch firstSendEntered; private CountDownLatch releaseFirstSend; @@ -280,6 +304,7 @@ public CompletableFuture send(Request request) { stalled = null; return result; } + if (request.path().endsWith("/result") && resultSubmission != null) return resultSubmission; if (request.path().endsWith("/operations")) { if (operationClaim != null) return operationClaim; return CompletableFuture.completedFuture(new Response(204, "")); From 87a9810cf19fd6dc86913a80d20a3d4519d0d731 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 19:26:11 -0600 Subject: [PATCH 62/73] Persist and reconcile proxy Control results --- .../votingplugin/proxy/VotingPluginProxy.java | 5 + .../proxy/control/ControlConnector.java | 166 ++++++++++++++---- .../control/ProxyControlResultStore.java | 161 +++++++++++++++++ .../proxy/control/ControlConnectorTest.java | 24 +++ .../control/ProxyControlResultStoreTest.java | 50 ++++++ 5 files changed, 370 insertions(+), 36 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index 9245c6474a..de9676118c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1593,6 +1593,11 @@ private void restartControlServicesAsync() { } } + /** Rebuilds a recovery connector from current settings after its durable result is acknowledged. */ + public final void restartControlServicesAfterRecovery() { + restartControlServicesAsync(); + } + private void stopControlServices(boolean waitForHosted) { synchronized (controlLifecycleLock) { stopControlServicesLocked(waitForHosted); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 07fed482a3..62b3f181cd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -34,6 +34,8 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.presence.BackendPresenceStatus; +import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.Route; +import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.StoredResult; import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; import com.bencodez.votingplugin.util.ControlCredentialFile; import com.google.gson.JsonArray; @@ -62,7 +64,11 @@ public final class ControlConnector implements AutoCloseable { private final UUID sessionId; private final LongSupplier jitterSource; private final ProxyRoutingConfigurationService configurationService; - private final Map completedTasks = new LinkedHashMap<>(); + private final Path dataDirectory; + private final Route route; + private final boolean recovering; + private final Runnable recoveryComplete; + private final Map completedTasks = new LinkedHashMap<>(); private final Object operationLifecycle = new Object(); private final AtomicBoolean inFlight = new AtomicBoolean(); private Runnable deferredReplacement; @@ -79,12 +85,21 @@ public final class ControlConnector implements AutoCloseable { public ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, Supplier> snapshotSource, Consumer logger, UUID sessionId, LongSupplier jitterSource) { - this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, null); + this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, null, + null, null, false, null); } ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, Supplier> snapshotSource, Consumer logger, UUID sessionId, LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService) { + this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, configurationService, + null, null, false, null); + } + + private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, + Supplier> snapshotSource, Consumer logger, UUID sessionId, + LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService, Path dataDirectory, + Route route, boolean recovering, Runnable recoveryComplete) { this.settings = Objects.requireNonNull(settings, "settings"); this.scheduler = Objects.requireNonNull(scheduler, "scheduler"); this.transport = Objects.requireNonNull(transport, "transport"); @@ -93,30 +108,44 @@ public ControlConnector(Settings settings, ScheduledExecutorService scheduler, T this.sessionId = Objects.requireNonNull(sessionId, "sessionId"); this.jitterSource = Objects.requireNonNull(jitterSource, "jitterSource"); this.configurationService = configurationService; + this.dataDirectory = dataDirectory; + this.route = route; + this.recovering = recovering; + this.recoveryComplete = recoveryComplete; } /** Creates the production connector without reading a credential when the feature is disabled. */ public static ControlConnector create(VotingPluginProxy proxy) throws IOException { - VotingPluginProxyConfig config = proxy.getConfig(); - if (!config.getControlEnabled()) { - return null; - } - String configuredNodeId = config.getControlNodeId(); - String nodeId = configuredNodeId == null || configuredNodeId.isBlank() - ? config.getProxyServerName() : configuredNodeId.trim(); Path dataDirectory = proxy.getDataFolderPlugin().toPath().toAbsolutePath().normalize(); - String credentialName = config.getControlCredentialFile(); - if (credentialName == null || credentialName.isBlank()) { - throw new IllegalArgumentException("Control.CredentialFile must be set"); + ProxyControlResultStore.State recovered = ProxyControlResultStore.load(dataDirectory); + VotingPluginProxyConfig config = proxy.getConfig(); + Settings settings; + Route route; + String credentialName; + boolean recovering = recovered != null; + if (recovered != null) { + route = recovered.route(); + settings = settings(route); + credentialName = route.credentialFile(); + } else { + if (!config.getControlEnabled()) return null; + String configuredNodeId = config.getControlNodeId(); + String nodeId = configuredNodeId == null || configuredNodeId.isBlank() + ? config.getProxyServerName() : configuredNodeId.trim(); + credentialName = config.getControlCredentialFile(); + if (credentialName == null || credentialName.isBlank()) { + throw new IllegalArgumentException("Control.CredentialFile must be set"); + } + String endpointValue = config.getControlEndpoint(); + if (endpointValue == null || endpointValue.isBlank()) { + throw new IllegalArgumentException("Control.Endpoint must be set"); + } + settings = new Settings(nodeId, nodeId, proxy.getProxyPlatform(), proxy.getPluginVersion(), + URI.create(endpointValue.trim()), config.getControlHeartbeatSeconds(), + config.getControlConnectTimeoutMillis(), config.getControlRequestTimeoutMillis()); + route = route(settings, credentialName); } String credential = ControlCredentialFile.read(dataDirectory, credentialName); - String endpointValue = config.getControlEndpoint(); - if (endpointValue == null || endpointValue.isBlank()) { - throw new IllegalArgumentException("Control.Endpoint must be set"); - } - Settings settings = new Settings(nodeId, nodeId, proxy.getProxyPlatform(), proxy.getPluginVersion(), - URI.create(endpointValue.trim()), config.getControlHeartbeatSeconds(), - config.getControlConnectTimeoutMillis(), config.getControlRequestTimeoutMillis()); HttpControlTransport transport = new HttpControlTransport(settings.endpoint(), credential, settings.connectTimeoutMillis(), settings.requestTimeoutMillis()); Supplier> snapshot = () -> { @@ -136,7 +165,9 @@ public static ControlConnector create(VotingPluginProxy proxy) throws IOExceptio }; ControlConnector connector = new ControlConnector(settings, proxy.getScheduler(), transport, snapshot, message -> proxy.log("[Control] " + message), UUID.randomUUID(), - () -> ThreadLocalRandom.current().nextLong(), new ProxyRoutingConfigurationService(proxy)); + () -> ThreadLocalRandom.current().nextLong(), new ProxyRoutingConfigurationService(proxy), dataDirectory, + route, recovering, proxy::restartControlServicesAfterRecovery); + if (recovered != null) connector.completedTasks.putAll(recovered.results()); return connector; } @@ -220,6 +251,7 @@ void cycle() { return presence.thenCompose(responseBody -> { handlePresenceResponse(responseBody); if (!configurationAccepted) return CompletableFuture.completedFuture(null); + if (hasCompletedTask()) return submitCompletedResult(); CompletableFuture claim = transport.send(claimRequest()); activeRequest = claim; return claim.thenCompose(this::handleClaimResponse); @@ -391,25 +423,64 @@ private CompletableFuture handleClaimResponse(Response response) { requireSuccess(response); JsonObject task = parseObject(response.body); UUID operationId = UUID.fromString(requireString(task, "operationId")); - TaskResult result; + StoredResult result; synchronized (operationLifecycle) { result = completedTasks.get(operationId); } if (result == null) { - result = executeTask(task); + TaskResult executed = executeTask(task); + result = new StoredResult(executed.json()); synchronized (operationLifecycle) { completedTasks.put(operationId, result); - while (completedTasks.size() > 128) completedTasks.remove(completedTasks.keySet().iterator().next()); } + persistCompleted(); + } + return submitCompletedResult(operationId, result); + } + + private boolean hasCompletedTask() { + synchronized (operationLifecycle) { return !completedTasks.isEmpty(); } + } + + private CompletableFuture submitCompletedResult() { + Map.Entry pending; + synchronized (operationLifecycle) { + pending = completedTasks.entrySet().stream().findFirst().orElse(null); } + if (pending == null) return CompletableFuture.completedFuture(null); + persistCompleted(); + return submitCompletedResult(pending.getKey(), pending.getValue()); + } + + private CompletableFuture submitCompletedResult(UUID operationId, StoredResult result) { CompletableFuture submitted = transport.send(resultRequest(operationId, result)); activeRequest = submitted; return submitted.thenAccept(resultResponse -> { requireSuccess(resultResponse); synchronized (operationLifecycle) { completedTasks.remove(operationId); } + try { + persistCompleted(); + } catch (RuntimeException failure) { + synchronized (operationLifecycle) { completedTasks.put(operationId, result); } + throw failure; + } + boolean drained; + synchronized (operationLifecycle) { drained = completedTasks.isEmpty(); } + if (recovering && drained && recoveryComplete != null) recoveryComplete.run(); }); } + private void persistCompleted() { + if (dataDirectory == null || route == null) return; + Map snapshot; + synchronized (operationLifecycle) { snapshot = new LinkedHashMap<>(completedTasks); } + try { + ProxyControlResultStore.save(dataDirectory, route, snapshot); + } catch (IOException e) { + throw new UnavailableException(e); + } + } + private TaskResult executeTask(JsonObject task) { if (configurationService == null) return TaskResult.failure("UNSUPPORTED", "Configuration control is unavailable"); String type = requireString(task, "type"); @@ -445,23 +516,28 @@ private static ProxyRoutingConfiguration parseConfiguration(JsonObject body) { return new ProxyRoutingConfiguration(body.get("sendVotesToAllServers").getAsBoolean(), blocked); } - private Request resultRequest(UUID operationId, TaskResult result) { - JsonObject body = new JsonObject(); + private Request resultRequest(UUID operationId, StoredResult result) { + JsonObject body = result.result().deepCopy(); body.addProperty("sessionId", sessionId.toString()); - body.addProperty("success", result.success); - body.addProperty("code", result.code); - body.addProperty("message", result.message); - if (result.revision != null) body.addProperty("revision", result.revision); - if (result.configuration != null) body.add("configuration", configurationJson(result.configuration)); - JsonArray changes = new JsonArray(); - result.changes.forEach(changes::add); - body.add("changes", changes); - body.addProperty("reloaded", result.reloaded); - body.addProperty("rolledBack", result.rolledBack); return new Request("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + "/result", body.toString()); } + private static Route route(Settings settings, String credentialFile) { + return new Route(settings.nodeId(), settings.displayName(), settings.platform(), settings.pluginVersion(), + settings.endpoint(), credentialFile, settings.heartbeatSeconds(), settings.connectTimeoutMillis(), + settings.requestTimeoutMillis()); + } + + private static Settings settings(Route route) { + if (route.credentialFile() == null || route.credentialFile().isBlank() + || route.credentialFile().length() > 2048) { + throw new IllegalArgumentException("Control.CredentialFile is invalid"); + } + return new Settings(route.nodeId(), route.displayName(), route.platform(), route.pluginVersion(), route.endpoint(), + route.heartbeatSeconds(), route.connectTimeoutMillis(), route.requestTimeoutMillis()); + } + private static JsonObject configurationJson(ProxyRoutingConfiguration configuration) { JsonObject body = new JsonObject(); body.addProperty("sendVotesToAllServers", configuration.sendVotesToAllServers()); @@ -637,6 +713,21 @@ public record Response(int statusCode, String body) { } private record TaskResult(boolean success, String code, String message, String revision, ProxyRoutingConfiguration configuration, List changes, boolean reloaded, boolean rolledBack) { + private JsonObject json() { + JsonObject body = new JsonObject(); + body.addProperty("success", success); + body.addProperty("code", code); + body.addProperty("message", message); + if (revision != null) body.addProperty("revision", revision); + if (configuration != null) body.add("configuration", configurationJson(configuration)); + JsonArray listed = new JsonArray(); + changes.forEach(listed::add); + body.add("changes", listed); + body.addProperty("reloaded", reloaded); + body.addProperty("rolledBack", rolledBack); + return body; + } + private static TaskResult success(String revision, ProxyRoutingConfiguration configuration, List changes, boolean reloaded) { return new TaskResult(true, "OK", "Operation completed", revision, configuration, changes, reloaded, false); @@ -687,7 +778,10 @@ private static final class ProtocolException extends ControlFailure { } @SuppressWarnings("serial") private static final class RegistryLostException extends ControlFailure { } @SuppressWarnings("serial") - private static final class UnavailableException extends ControlFailure { } + private static final class UnavailableException extends ControlFailure { + private UnavailableException() { } + private UnavailableException(Throwable cause) { super.initCause(cause); } + } @SuppressWarnings("serial") private static final class MalformedResponseException extends ControlFailure { } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java new file mode 100644 index 0000000000..9e9dcb91bd --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java @@ -0,0 +1,161 @@ +package com.bencodez.votingplugin.proxy.control; + +import java.io.IOException; +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.charset.StandardCharsets; +import java.nio.file.AtomicMoveNotSupportedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Set; +import java.util.UUID; + +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; + +/** Durable result journal for proxy-routing operations accepted by a specific Control node. */ +final class ProxyControlResultStore { + private static final int VERSION = 1; + private static final int MAX_BYTES = 256 * 1024; + private static final int MAX_RESULTS = 128; + private static final String FILE_NAME = ".control-proxy-pending-results.json"; + + private ProxyControlResultStore() { } + + static State load(Path dataDirectory) throws IOException { + Path target = target(dataDirectory); + if (!Files.exists(target, LinkOption.NOFOLLOW_LINKS)) return null; + if (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target) + || Files.size(target) > MAX_BYTES) { + throw new IOException("Control proxy-result journal is unsafe or too large"); + } + byte[] bytes; + try (SeekableByteChannel channel = Files.newByteChannel(target, + Set.of(StandardOpenOption.READ, LinkOption.NOFOLLOW_LINKS))) { + ByteBuffer buffer = ByteBuffer.allocate((int) Files.size(target) + 1); + while (channel.read(buffer) >= 0 && buffer.hasRemaining()) { } + if (!buffer.hasRemaining()) throw new IOException("Control proxy-result journal is too large"); + buffer.flip(); + bytes = new byte[buffer.remaining()]; + buffer.get(bytes); + } + try { + JsonElement parsed = JsonParser.parseString(new String(bytes, StandardCharsets.UTF_8)); + if (!parsed.isJsonObject()) throw invalid(); + JsonObject root = parsed.getAsJsonObject(); + if (integer(root, "version") != VERSION) throw invalid(); + JsonObject routeJson = object(root, "route"); + Route route = new Route(string(routeJson, "nodeId"), string(routeJson, "displayName"), + string(routeJson, "platform"), string(routeJson, "pluginVersion"), + URI.create(string(routeJson, "endpoint")), string(routeJson, "credentialFile"), + integer(routeJson, "heartbeatSeconds"), integer(routeJson, "connectTimeoutMillis"), + integer(routeJson, "requestTimeoutMillis")); + JsonArray listed = array(root, "results"); + if (listed.size() == 0 || listed.size() > MAX_RESULTS) throw invalid(); + Map results = new LinkedHashMap<>(); + for (JsonElement element : listed) { + if (!element.isJsonObject()) throw invalid(); + JsonObject item = element.getAsJsonObject(); + UUID operationId = UUID.fromString(string(item, "operationId")); + StoredResult previous = results.put(operationId, + new StoredResult(object(item, "result").deepCopy())); + if (previous != null) throw invalid(); + } + return new State(route, Map.copyOf(results)); + } catch (RuntimeException e) { + throw new IOException("Control proxy-result journal is malformed", e); + } + } + + static void save(Path dataDirectory, Route route, Map results) throws IOException { + Path target = target(dataDirectory); + if (results.isEmpty()) { + if (Files.isSymbolicLink(target)) throw new IOException("Control proxy-result journal is unsafe"); + Files.deleteIfExists(target); + return; + } + if (results.size() > MAX_RESULTS) throw new IOException("Too many pending Control proxy results"); + if (Files.exists(target, LinkOption.NOFOLLOW_LINKS) + && (!Files.isRegularFile(target, LinkOption.NOFOLLOW_LINKS) || Files.isSymbolicLink(target))) { + throw new IOException("Control proxy-result journal is unsafe"); + } + JsonObject root = new JsonObject(); + root.addProperty("version", VERSION); + JsonObject routeJson = new JsonObject(); + routeJson.addProperty("nodeId", route.nodeId()); + routeJson.addProperty("displayName", route.displayName()); + routeJson.addProperty("platform", route.platform()); + routeJson.addProperty("pluginVersion", route.pluginVersion()); + routeJson.addProperty("endpoint", route.endpoint().toString()); + routeJson.addProperty("credentialFile", route.credentialFile()); + routeJson.addProperty("heartbeatSeconds", route.heartbeatSeconds()); + routeJson.addProperty("connectTimeoutMillis", route.connectTimeoutMillis()); + routeJson.addProperty("requestTimeoutMillis", route.requestTimeoutMillis()); + root.add("route", routeJson); + JsonArray listed = new JsonArray(); + results.forEach((operationId, result) -> { + JsonObject item = new JsonObject(); + item.addProperty("operationId", operationId.toString()); + item.add("result", result.result().deepCopy()); + listed.add(item); + }); + root.add("results", listed); + byte[] bytes = root.toString().getBytes(StandardCharsets.UTF_8); + if (bytes.length > MAX_BYTES) throw new IOException("Control proxy-result journal is too large"); + Files.createDirectories(dataDirectory); + Path staging = Files.createTempFile(dataDirectory, ".control-proxy-results-", ".json"); + try { + Files.write(staging, bytes, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + try { + Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + } catch (AtomicMoveNotSupportedException e) { + Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING); + } + } finally { + Files.deleteIfExists(staging); + } + } + + private static Path target(Path dataDirectory) { + return dataDirectory.toAbsolutePath().normalize().resolve(FILE_NAME); + } + + private static JsonObject object(JsonObject object, String name) { + if (!object.has(name) || !object.get(name).isJsonObject()) throw invalid(); + return object.getAsJsonObject(name); + } + + private static JsonArray array(JsonObject object, String name) { + if (!object.has(name) || !object.get(name).isJsonArray()) throw invalid(); + return object.getAsJsonArray(name); + } + + private static String string(JsonObject object, String name) { + if (!object.has(name) || !object.get(name).isJsonPrimitive()) throw invalid(); + String value = object.get(name).getAsString(); + if (value == null || value.isBlank() || value.length() > 2048) throw invalid(); + return value; + } + + private static int integer(JsonObject object, String name) { + if (!object.has(name) || !object.get(name).isJsonPrimitive()) throw invalid(); + return object.get(name).getAsInt(); + } + + private static IllegalArgumentException invalid() { + return new IllegalArgumentException("invalid proxy-result journal"); + } + + record Route(String nodeId, String displayName, String platform, String pluginVersion, URI endpoint, + String credentialFile, int heartbeatSeconds, int connectTimeoutMillis, int requestTimeoutMillis) { } + record StoredResult(JsonObject result) { } + record State(Route route, Map results) { } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 6038b67102..43a877e851 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -220,6 +220,30 @@ class ControlConnectorTest { assertTrue(replacementStarted.get()); } + @Test void lostResultResponseIsResubmittedBeforeAnotherOperationClaim() { + connector.close(); + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L, + new ProxyRoutingConfigurationService(new NoOpPlatform())); + transport.acceptConfiguration = true; + transport.operationClaim = CompletableFuture.completedFuture(new Response(200, + "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"type\":\"APPLY\",\"expectedRevision\":\"" + current.revision() + "\"," + + "\"configuration\":{\"sendVotesToAllServers\":true,\"blockedServers\":[]}}")); + transport.resultSubmission = new CompletableFuture<>(); + connector.cycle(); + transport.resultSubmission.completeExceptionally(new java.io.IOException("response lost")); + assertEquals(Status.UNAVAILABLE, connector.status()); + + transport.operationClaim = CompletableFuture.completedFuture(new Response(204, "")); + transport.resultSubmission = CompletableFuture.completedFuture(new Response(200, "{}")); + connector.cycle(); + + assertEquals(2, transport.requests.stream().filter(request -> request.path().endsWith("/result")).count()); + assertEquals(1, transport.requests.stream().filter(request -> request.path().endsWith("/operations")).count()); + } + @Test void closeBeforeOperationPublicationPreventsTheRequestChainFromStarting() throws Exception { connector.close(); transport.firstSendEntered = new CountDownLatch(1); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java new file mode 100644 index 0000000000..48cd074a4b --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java @@ -0,0 +1,50 @@ +package com.bencodez.votingplugin.proxy.control; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.UUID; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.Route; +import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.StoredResult; +import com.google.gson.JsonObject; + +class ProxyControlResultStoreTest { + @TempDir Path directory; + + @Test void proxyResultAndOriginSurviveRestartUntilControlAcknowledgesThem() throws Exception { + UUID operationId = UUID.fromString("00000000-0000-0000-0000-000000000099"); + Route route = new Route("proxy-old", "Proxy Old", "VELOCITY", "7.1.2", + URI.create("https://control.example:8443"), "old-credential.txt", 30, 3000, 5000); + JsonObject result = new JsonObject(); + result.addProperty("success", true); + result.addProperty("revision", "applied-revision"); + Map pending = new LinkedHashMap<>(); + pending.put(operationId, new StoredResult(result)); + + ProxyControlResultStore.save(directory, route, pending); + ProxyControlResultStore.State recovered = ProxyControlResultStore.load(directory); + + assertEquals(route, recovered.route()); + assertEquals("applied-revision", recovered.results().get(operationId).result().get("revision").getAsString()); + ProxyControlResultStore.save(directory, route, Map.of()); + assertFalse(Files.exists(directory.resolve(".control-proxy-pending-results.json"))); + } + + @Test void symbolicProxyResultJournalIsRejected() throws Exception { + Path external = directory.resolve("external.json"); + Files.writeString(external, "{}"); + Files.createSymbolicLink(directory.resolve(".control-proxy-pending-results.json"), external); + + assertThrows(java.io.IOException.class, () -> ProxyControlResultStore.load(directory)); + } +} From 2a748650525d59de4166f277c147794839f2c496 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 19:34:55 -0600 Subject: [PATCH 63/73] Recover proxy results before honoring disablement --- .../java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java | 1 - 1 file changed, 1 deletion(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java index de9676118c..ab0dab545a 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/VotingPluginProxy.java @@ -1614,7 +1614,6 @@ private void startControlServicesLocked() { logSevere("[Control Host] configuration is invalid; VotingPlugin remains unaffected"); } } - if (!getConfig().getControlEnabled()) return; try { controlConnector = ControlConnector.create(this); if (controlConnector != null) controlConnector.start(); From b09600941de0923b270de01b44eaf8f3e0e8023d Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 19:58:32 -0600 Subject: [PATCH 64/73] Align backend Control payload bounds --- .../control/BackendControlConnector.java | 2 +- .../control/BackendControlResultStore.java | 3 ++- .../BackendControlResultStoreTest.java | 19 +++++++++++++++++++ 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 59f429f0cd..b04dbed767 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -40,7 +40,7 @@ /** Optional outbound Bukkit node connector for full configuration-file and quick-setup control. */ public final class BackendControlConnector implements AutoCloseable { private static final int PROTOCOL_VERSION = 1; - private static final int MAX_RESPONSE_BYTES = 768 * 1024; + private static final int MAX_RESPONSE_BYTES = 4 * 1024 * 1024; private static final long SHUTDOWN_TIMEOUT_SECONDS = 65; private static final Pattern NODE_ID = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,63}"); private static final Set CAPABILITIES = Set.of("config.files.v1", "config.quick-setup.v1"); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java index 193e096a3a..36c5bfedd6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java @@ -24,7 +24,8 @@ /** Durable result journal used to make locally applied Control operations restart-safe. */ final class BackendControlResultStore { private static final int VERSION = 1; - private static final int MAX_BYTES = 1024 * 1024; + // A valid 512 KiB YAML result may expand up to sixfold when characters require JSON unicode escaping. + private static final int MAX_BYTES = 4 * 1024 * 1024; private static final int MAX_RESULTS = 128; private static final String FILE_NAME = ".control-pending-results.json"; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java index 5162f38de6..1737d9b074 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java @@ -47,4 +47,23 @@ class BackendControlResultStoreTest { assertThrows(java.io.IOException.class, () -> BackendControlResultStore.load(directory)); } + + @Test void maximumEscapeHeavyConfigurationFitsTheEncodedJournal() throws Exception { + Route route = new Route("backend-old", URI.create("https://control.example:8443"), "credential.txt", + 30, 3000, 10000); + JsonObject configuration = new JsonObject(); + configuration.addProperty("domain", "file"); + configuration.addProperty("fileName", "Config.yml"); + configuration.addProperty("content", "\\".repeat(BackendConfigurationService.MAX_CONTENT_BYTES)); + JsonObject result = new JsonObject(); + result.addProperty("success", true); + result.add("configuration", configuration); + + BackendControlResultStore.save(directory, route, + Map.of(UUID.fromString("00000000-0000-0000-0000-000000000099"), new StoredResult(result, false))); + + assertEquals(BackendConfigurationService.MAX_CONTENT_BYTES, + BackendControlResultStore.load(directory).results().values().iterator().next().result() + .getAsJsonObject("configuration").get("content").getAsString().length()); + } } From 640e603be55afcd596432b15e874c1341658de7f Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 20:24:06 -0600 Subject: [PATCH 65/73] Make configuration apply recovery crash-safe --- .../control/BackendConfigurationService.java | 13 +++ .../control/BackendControlConnector.java | 97 ++++++++++++++++--- .../control/BackendControlResultStore.java | 9 +- .../proxy/control/ControlConnector.java | 63 +++++++++++- .../control/ProxyControlResultStore.java | 6 +- .../BackendControlConnectorProtocolTest.java | 8 ++ .../BackendControlResultStoreTest.java | 20 +++- .../proxy/control/ControlConnectorTest.java | 36 +++++++ .../control/ProxyControlResultStoreTest.java | 16 ++- 9 files changed, 241 insertions(+), 27 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 8e2df21e15..f50fa05325 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -60,6 +60,11 @@ public Preview preview(String fileName, String proposedContent) throws IOExcepti return preview(fileName, proposedContent, current); } + Document proposedDocument(Preview preview) { + return new Document(preview.fileName(), mask(parse(preview.resolvedContent())), + revision(preview.resolvedContent())); + } + public ApplyResult apply(String fileName, String proposedContent, String expectedRevision) throws IOException { Path target = resolve(fileName); String current = readRaw(target, false); @@ -163,6 +168,14 @@ public QuickPreview previewQuickSetup(String preset, Map options return new QuickPreview(proposal, revision(current), changes(parse(current), parse(proposal.content()))); } + String proposedQuickSetupRevision(QuickPreview preview) { + return revision(preview.proposal().content()); + } + + String currentQuickSetupRevision(String preset) throws IOException { + return read(quickSetupFile(preset)).revision(); + } + public ApplyResult applyQuickSetup(String preset, Map options, String expectedRevision) throws IOException { String fileName = quickSetupFile(preset); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index b04dbed767..09347e4b79 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -237,9 +237,20 @@ private void claimAndExecute() throws Exception { synchronized (completed) { result = completed.get(operationId); } + if (result != null) { + if (!result.committed() && !anticipatedResultIsInstalled(result)) { + result = null; + } else { + result = committedForAttempt(result, string(task, "attemptId")); + synchronized (completed) { completed.put(operationId, result); } + persistCompleted(); + } + } if (result == null) { - TaskResult executed = execute(task); - result = new StoredResult(executed.json(), executed.restartConnector()); + TaskResult executed = execute(operationId, task); + JsonObject resultJson = executed.json(); + resultJson.addProperty("attemptId", string(task, "attemptId")); + result = new StoredResult(resultJson, executed.restartConnector(), true); synchronized (completed) { completed.put(operationId, result); } @@ -251,20 +262,27 @@ private void claimAndExecute() throws Exception { private boolean submitCompletedResult() throws Exception { Map.Entry pending; synchronized (completed) { - pending = completed.entrySet().stream().findFirst().orElse(null); + pending = completed.entrySet().stream().filter(entry -> entry.getValue().committed()).findFirst().orElse(null); } if (pending == null) return false; persistCompleted(); - submitCompletedResult(pending.getKey(), pending.getValue()); - return true; + return submitCompletedResult(pending.getKey(), pending.getValue()); } - private void submitCompletedResult(UUID operationId, StoredResult submitted) throws Exception { + private boolean submitCompletedResult(UUID operationId, StoredResult submitted) throws Exception { JsonObject resultBody = submitted.result().deepCopy(); resultBody.addProperty("sessionId", sessionId.toString()); + Response response = send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId + + "/result", resultBody); + if (taskLeaseExpired(response)) { + synchronized (completed) { + completed.put(operationId, new StoredResult(submitted.result(), submitted.restartConnector(), false)); + } + persistCompleted(); + return false; + } afterResultAcknowledged( - () -> requireObject(send("POST", "/api/v1/nodes/" + settings.nodeId() + "/operations/" + operationId - + "/result", resultBody), 200), + () -> requireObject(response, 200), () -> { synchronized (completed) { completed.remove(operationId); } try { @@ -279,6 +297,17 @@ private void submitCompletedResult(UUID operationId, StoredResult submitted) thr plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); } }); + return true; + } + + static boolean taskLeaseExpired(Response response) { + if (response.status() != 409) return false; + try { + JsonObject error = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("error"); + return error != null && error.has("code") && "TASK_LEASE_EXPIRED".equals(error.get("code").getAsString()); + } catch (RuntimeException ignored) { + return false; + } } private void persistCompleted() throws IOException { @@ -287,21 +316,51 @@ private void persistCompleted() throws IOException { BackendControlResultStore.save(dataDirectory, settings.route(), snapshot); } + private void persistIntent(UUID operationId, TaskResult anticipated, String attemptId) throws IOException { + JsonObject result = anticipated.json(); + result.addProperty("attemptId", attemptId); + synchronized (completed) { + completed.put(operationId, new StoredResult(result, anticipated.restartConnector(), false)); + } + persistCompleted(); + } + + private boolean anticipatedResultIsInstalled(StoredResult pending) throws IOException { + JsonObject result = pending.result(); + if (!result.has("revision") || !result.has("configuration")) return false; + String revision = result.get("revision").getAsString(); + JsonObject configuration = result.getAsJsonObject("configuration"); + String domain = string(configuration, "domain"); + if ("file".equals(domain)) { + return revision.equals(configurations.read(string(configuration, "fileName")).revision()); + } + if ("quick-setup".equals(domain)) { + return revision.equals(configurations.currentQuickSetupRevision(string(configuration, "preset"))); + } + return false; + } + + private static StoredResult committedForAttempt(StoredResult pending, String attemptId) { + JsonObject result = pending.result().deepCopy(); + result.addProperty("attemptId", attemptId); + return new StoredResult(result, pending.restartConnector(), true); + } + static void afterResultAcknowledged(ResultSubmission submission, ResultAcknowledgement acknowledged) throws Exception { submission.submit(); acknowledged.acknowledge(); } - private TaskResult execute(JsonObject task) { + private TaskResult execute(UUID operationId, JsonObject task) { try { String type = string(task, "type"); JsonObject configuration = task.getAsJsonObject("configuration"); String domain = string(configuration, "domain"); - if ("file".equals(domain)) return executeFile(type, configuration, task); + if ("file".equals(domain)) return executeFile(operationId, type, configuration, task); if ("quick-setup".equals(domain)) { if (!quickSetupsAccepted) return TaskResult.failure("UNSUPPORTED_TASK", "Quick setups were not negotiated"); - return executeQuick(type, configuration, task); + return executeQuick(operationId, type, configuration, task); } return TaskResult.failure("UNSUPPORTED_TASK", "Configuration domain is unsupported"); } catch (BackendConfigurationService.StaleRevisionException e) { @@ -315,7 +374,8 @@ private TaskResult execute(JsonObject task) { } } - private TaskResult executeFile(String type, JsonObject configuration, JsonObject task) throws IOException { + private TaskResult executeFile(UUID operationId, String type, JsonObject configuration, JsonObject task) + throws IOException { String fileName = string(configuration, "fileName"); if ("READ".equals(type)) { BackendConfigurationService.Document document = configurations.read(fileName); @@ -329,6 +389,10 @@ private TaskResult executeFile(String type, JsonObject configuration, JsonObject return TaskResult.file(current, preview.changes(), false, false, false); } if ("APPLY".equals(type)) { + BackendConfigurationService.Preview preview = configurations.preview(fileName, content); + persistIntent(operationId, + TaskResult.file(configurations.proposedDocument(preview), preview.changes(), true, false, + "Config.yml".equals(fileName)), string(task, "attemptId")); BackendConfigurationService.ApplyResult applied = configurations.apply(fileName, content, string(task, "expectedRevision")); return TaskResult.file(applied.document(), applied.changes(), true, applied.rolledBack(), @@ -337,7 +401,8 @@ private TaskResult executeFile(String type, JsonObject configuration, JsonObject return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); } - private TaskResult executeQuick(String type, JsonObject configuration, JsonObject task) throws IOException { + private TaskResult executeQuick(UUID operationId, String type, JsonObject configuration, JsonObject task) + throws IOException { if ("READ".equals(type)) return TaskResult.failure("UNSUPPORTED_TASK", "Quick setups cannot be read"); String preset = string(configuration, "preset"); Map options = options(configuration.getAsJsonObject("options")); @@ -346,6 +411,10 @@ private TaskResult executeQuick(String type, JsonObject configuration, JsonObjec return TaskResult.quick(preset, options, preview.revision(), preview.changes(), false); } if ("APPLY".equals(type)) { + BackendConfigurationService.QuickPreview preview = configurations.previewQuickSetup(preset, options); + persistIntent(operationId, + TaskResult.quick(preset, options, configurations.proposedQuickSetupRevision(preview), preview.changes(), + true, "Config.yml".equals(preview.proposal().fileName())), string(task, "attemptId")); BackendConfigurationService.ApplyResult applied = configurations.applyQuickSetup(preset, options, string(task, "expectedRevision")); return TaskResult.quick(preset, options, applied.document().revision(), applied.changes(), true, @@ -488,7 +557,7 @@ private Route route() { requestTimeoutMillis); } } - private record Response(int status, String body) { } + record Response(int status, String body) { } @FunctionalInterface interface ResultSubmission { void submit() throws Exception; } @FunctionalInterface diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java index 36c5bfedd6..dfdc6dd82d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java @@ -65,9 +65,11 @@ static State load(Path dataDirectory) throws IOException { JsonObject item = element.getAsJsonObject(); UUID operationId = UUID.fromString(string(item, "operationId")); JsonObject result = object(item, "result").deepCopy(); - if (!item.has("restartConnector") || !item.get("restartConnector").isJsonPrimitive()) throw invalid(); + if (!item.has("restartConnector") || !item.get("restartConnector").isJsonPrimitive() + || !item.has("committed") || !item.get("committed").isJsonPrimitive()) throw invalid(); StoredResult previous = results.put(operationId, - new StoredResult(result, item.get("restartConnector").getAsBoolean())); + new StoredResult(result, item.get("restartConnector").getAsBoolean(), + item.get("committed").getAsBoolean())); if (previous != null) throw invalid(); } return new State(route, Map.copyOf(results)); @@ -104,6 +106,7 @@ static void save(Path dataDirectory, Route route, Map result item.addProperty("operationId", operationId.toString()); item.add("result", result.result().deepCopy()); item.addProperty("restartConnector", result.restartConnector()); + item.addProperty("committed", result.committed()); listed.add(item); }); root.add("results", listed); @@ -155,6 +158,6 @@ private static IllegalArgumentException invalid() { record Route(String nodeId, URI endpoint, String credentialFile, int heartbeatSeconds, int connectTimeoutMillis, int requestTimeoutMillis) { } - record StoredResult(JsonObject result, boolean restartConnector) { } + record StoredResult(JsonObject result, boolean restartConnector, boolean committed) { } record State(Route route, Map results) { } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 62b3f181cd..ff3a4662f5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -427,9 +427,20 @@ private CompletableFuture handleClaimResponse(Response response) { synchronized (operationLifecycle) { result = completedTasks.get(operationId); } + if (result != null) { + if (!result.committed() && !anticipatedResultIsInstalled(result)) { + result = null; + } else { + result = committedForAttempt(result, requireString(task, "attemptId")); + synchronized (operationLifecycle) { completedTasks.put(operationId, result); } + persistCompleted(); + } + } if (result == null) { - TaskResult executed = executeTask(task); - result = new StoredResult(executed.json()); + TaskResult executed = executeTask(operationId, task); + JsonObject resultJson = executed.json(); + resultJson.addProperty("attemptId", requireString(task, "attemptId")); + result = new StoredResult(resultJson, true); synchronized (operationLifecycle) { completedTasks.put(operationId, result); } @@ -439,13 +450,16 @@ private CompletableFuture handleClaimResponse(Response response) { } private boolean hasCompletedTask() { - synchronized (operationLifecycle) { return !completedTasks.isEmpty(); } + synchronized (operationLifecycle) { + return completedTasks.values().stream().anyMatch(StoredResult::committed); + } } private CompletableFuture submitCompletedResult() { Map.Entry pending; synchronized (operationLifecycle) { - pending = completedTasks.entrySet().stream().findFirst().orElse(null); + pending = completedTasks.entrySet().stream().filter(entry -> entry.getValue().committed()) + .findFirst().orElse(null); } if (pending == null) return CompletableFuture.completedFuture(null); persistCompleted(); @@ -456,6 +470,13 @@ private CompletableFuture submitCompletedResult(UUID operationId, StoredRe CompletableFuture submitted = transport.send(resultRequest(operationId, result)); activeRequest = submitted; return submitted.thenAccept(resultResponse -> { + if (taskLeaseExpired(resultResponse)) { + synchronized (operationLifecycle) { + completedTasks.put(operationId, new StoredResult(result.result(), false)); + } + persistCompleted(); + return; + } requireSuccess(resultResponse); synchronized (operationLifecycle) { completedTasks.remove(operationId); } try { @@ -470,6 +491,16 @@ private CompletableFuture submitCompletedResult(UUID operationId, StoredRe }); } + private static boolean taskLeaseExpired(Response response) { + if (response.statusCode != 409) return false; + try { + JsonObject error = parseObject(response.body).getAsJsonObject("error"); + return error != null && error.has("code") && "TASK_LEASE_EXPIRED".equals(error.get("code").getAsString()); + } catch (RuntimeException ignored) { + return false; + } + } + private void persistCompleted() { if (dataDirectory == null || route == null) return; Map snapshot; @@ -481,7 +512,27 @@ private void persistCompleted() { } } - private TaskResult executeTask(JsonObject task) { + private void persistIntent(UUID operationId, TaskResult anticipated, String attemptId) { + JsonObject result = anticipated.json(); + result.addProperty("attemptId", attemptId); + synchronized (operationLifecycle) { + completedTasks.put(operationId, new StoredResult(result, false)); + } + persistCompleted(); + } + + private boolean anticipatedResultIsInstalled(StoredResult pending) { + JsonObject result = pending.result(); + return result.has("revision") && result.get("revision").getAsString().equals(configurationService.read().revision()); + } + + private static StoredResult committedForAttempt(StoredResult pending, String attemptId) { + JsonObject result = pending.result().deepCopy(); + result.addProperty("attemptId", attemptId); + return new StoredResult(result, true); + } + + private TaskResult executeTask(UUID operationId, JsonObject task) { if (configurationService == null) return TaskResult.failure("UNSUPPORTED", "Configuration control is unavailable"); String type = requireString(task, "type"); try { @@ -492,6 +543,8 @@ private TaskResult executeTask(JsonObject task) { List changes = proposal.changesFrom(current); if ("PREVIEW".equals(type)) return TaskResult.success(current.revision(), current, changes, false); if (!"APPLY".equals(type)) return TaskResult.failure("UNSUPPORTED_TASK", "Task type is unsupported"); + persistIntent(operationId, TaskResult.success(proposal.revision(), proposal, changes, true), + requireString(task, "attemptId")); configurationService.apply(proposal, requireString(task, "expectedRevision")); ProxyRoutingConfiguration applied = configurationService.read(); return TaskResult.success(applied.revision(), applied, changes, true); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java index 9e9dcb91bd..6bce1705c7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java @@ -65,8 +65,9 @@ static State load(Path dataDirectory) throws IOException { if (!element.isJsonObject()) throw invalid(); JsonObject item = element.getAsJsonObject(); UUID operationId = UUID.fromString(string(item, "operationId")); + if (!item.has("committed") || !item.get("committed").isJsonPrimitive()) throw invalid(); StoredResult previous = results.put(operationId, - new StoredResult(object(item, "result").deepCopy())); + new StoredResult(object(item, "result").deepCopy(), item.get("committed").getAsBoolean())); if (previous != null) throw invalid(); } return new State(route, Map.copyOf(results)); @@ -105,6 +106,7 @@ static void save(Path dataDirectory, Route route, Map result JsonObject item = new JsonObject(); item.addProperty("operationId", operationId.toString()); item.add("result", result.result().deepCopy()); + item.addProperty("committed", result.committed()); listed.add(item); }); root.add("results", listed); @@ -156,6 +158,6 @@ private static IllegalArgumentException invalid() { record Route(String nodeId, String displayName, String platform, String pluginVersion, URI endpoint, String credentialFile, int heartbeatSeconds, int connectTimeoutMillis, int requestTimeoutMillis) { } - record StoredResult(JsonObject result) { } + record StoredResult(JsonObject result, boolean committed) { } record State(Route route, Map results) { } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index ebb451d867..7a30ef1105 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -17,6 +17,14 @@ import com.google.gson.JsonObject; class BackendControlConnectorProtocolTest { + @Test void onlyTheLeaseExpiryConflictRequestsAResultReclaim() { + assertTrue(BackendControlConnector.taskLeaseExpired(new BackendControlConnector.Response(409, + "{\"error\":{\"code\":\"TASK_LEASE_EXPIRED\"}}"))); + assertFalse(BackendControlConnector.taskLeaseExpired(new BackendControlConnector.Response(409, + "{\"error\":{\"code\":\"SESSION_MISMATCH\"}}"))); + assertFalse(BackendControlConnector.taskLeaseExpired(new BackendControlConnector.Response(500, "not-json"))); + } + @Test void registrationRequiresFileControlButAllowsQuickSetupToRemainOptional() { assertThrows(RuntimeException.class, () -> BackendControlConnector.requireFileCapability(false)); assertDoesNotThrow(() -> BackendControlConnector.requireFileCapability(true)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java index 1737d9b074..6b53217c99 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.URI; import java.nio.file.Files; @@ -29,17 +30,31 @@ class BackendControlResultStoreTest { result.addProperty("success", true); result.addProperty("revision", "applied-revision"); Map pending = new LinkedHashMap<>(); - pending.put(operationId, new StoredResult(result, true)); + pending.put(operationId, new StoredResult(result, true, true)); BackendControlResultStore.save(directory, route, pending); BackendControlResultStore.State recovered = BackendControlResultStore.load(directory); assertEquals(route, recovered.route()); assertEquals("applied-revision", recovered.results().get(operationId).result().get("revision").getAsString()); + assertTrue(recovered.results().get(operationId).committed()); BackendControlResultStore.save(directory, route, Map.of()); assertFalse(Files.exists(directory.resolve(".control-pending-results.json"))); } + @Test void writeAheadIntentRetainsItsUncommittedStateAcrossRestart() throws Exception { + UUID operationId = UUID.fromString("00000000-0000-0000-0000-000000000099"); + Route route = new Route("backend-old", URI.create("https://control.example:8443"), "old-credential.txt", + 30, 3000, 10000); + JsonObject result = new JsonObject(); + result.addProperty("revision", "anticipated-revision"); + + BackendControlResultStore.save(directory, route, + Map.of(operationId, new StoredResult(result, false, false))); + + assertFalse(BackendControlResultStore.load(directory).results().get(operationId).committed()); + } + @Test void symbolicPendingResultJournalIsRejected() throws Exception { Path external = directory.resolve("external.json"); Files.writeString(external, "{}"); @@ -60,7 +75,8 @@ class BackendControlResultStoreTest { result.add("configuration", configuration); BackendControlResultStore.save(directory, route, - Map.of(UUID.fromString("00000000-0000-0000-0000-000000000099"), new StoredResult(result, false))); + Map.of(UUID.fromString("00000000-0000-0000-0000-000000000099"), + new StoredResult(result, false, true))); assertEquals(BackendConfigurationService.MAX_CONTENT_BYTES, BackendControlResultStore.load(directory).results().values().iterator().next().result() diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 43a877e851..7e77faf43e 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -183,6 +183,7 @@ class ControlConnectorTest { try { CompletableFuture executing = CompletableFuture.runAsync(() -> transport.operationClaim.complete( new Response(200, "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"attemptId\":\"00000000-0000-0000-0000-000000000199\"," + "\"type\":\"APPLY\",\"expectedRevision\":\"" + current.revision() + "\"," + "\"configuration\":{\"sendVotesToAllServers\":true,\"blockedServers\":[]}}"))); assertTrue(persistEntered.await(2, TimeUnit.SECONDS)); @@ -207,6 +208,7 @@ class ControlConnectorTest { transport.acceptConfiguration = true; transport.operationClaim = CompletableFuture.completedFuture(new Response(200, "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"attemptId\":\"00000000-0000-0000-0000-000000000199\"," + "\"type\":\"APPLY\",\"expectedRevision\":\"" + current.revision() + "\"," + "\"configuration\":{\"sendVotesToAllServers\":true,\"blockedServers\":[]}}")); transport.resultSubmission = new CompletableFuture<>(); @@ -229,6 +231,7 @@ class ControlConnectorTest { transport.acceptConfiguration = true; transport.operationClaim = CompletableFuture.completedFuture(new Response(200, "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"attemptId\":\"00000000-0000-0000-0000-000000000199\"," + "\"type\":\"APPLY\",\"expectedRevision\":\"" + current.revision() + "\"," + "\"configuration\":{\"sendVotesToAllServers\":true,\"blockedServers\":[]}}")); transport.resultSubmission = new CompletableFuture<>(); @@ -244,6 +247,39 @@ class ControlConnectorTest { assertEquals(1, transport.requests.stream().filter(request -> request.path().endsWith("/operations")).count()); } + @Test void expiredResultLeaseIsReclaimedAndReboundBeforeResubmission() { + connector.close(); + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L, + new ProxyRoutingConfigurationService(new NoOpPlatform())); + transport.acceptConfiguration = true; + transport.operationClaim = CompletableFuture.completedFuture(new Response(200, + "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"attemptId\":\"00000000-0000-0000-0000-000000000199\"," + + "\"type\":\"APPLY\",\"expectedRevision\":\"" + current.revision() + "\"," + + "\"configuration\":{\"sendVotesToAllServers\":true,\"blockedServers\":[]}}")); + transport.resultSubmission = CompletableFuture.completedFuture(new Response(409, + "{\"error\":{\"code\":\"TASK_LEASE_EXPIRED\"}}")); + + connector.cycle(); + + transport.operationClaim = CompletableFuture.completedFuture(new Response(200, + "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"attemptId\":\"00000000-0000-0000-0000-000000000299\"," + + "\"type\":\"APPLY\",\"expectedRevision\":\"" + current.revision() + "\"," + + "\"configuration\":{\"sendVotesToAllServers\":true,\"blockedServers\":[]}}")); + transport.resultSubmission = CompletableFuture.completedFuture(new Response(200, "{}")); + connector.cycle(); + + List results = transport.requests.stream().filter(request -> request.path().endsWith("/result")).toList(); + assertEquals(2, results.size()); + assertEquals("00000000-0000-0000-0000-000000000199", + JsonParser.parseString(results.get(0).body()).getAsJsonObject().get("attemptId").getAsString()); + assertEquals("00000000-0000-0000-0000-000000000299", + JsonParser.parseString(results.get(1).body()).getAsJsonObject().get("attemptId").getAsString()); + } + @Test void closeBeforeOperationPublicationPreventsTheRequestChainFromStarting() throws Exception { connector.close(); transport.firstSendEntered = new CountDownLatch(1); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java index 48cd074a4b..e6bdddfad9 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.net.URI; import java.nio.file.Files; @@ -29,17 +30,30 @@ class ProxyControlResultStoreTest { result.addProperty("success", true); result.addProperty("revision", "applied-revision"); Map pending = new LinkedHashMap<>(); - pending.put(operationId, new StoredResult(result)); + pending.put(operationId, new StoredResult(result, true)); ProxyControlResultStore.save(directory, route, pending); ProxyControlResultStore.State recovered = ProxyControlResultStore.load(directory); assertEquals(route, recovered.route()); assertEquals("applied-revision", recovered.results().get(operationId).result().get("revision").getAsString()); + assertTrue(recovered.results().get(operationId).committed()); ProxyControlResultStore.save(directory, route, Map.of()); assertFalse(Files.exists(directory.resolve(".control-proxy-pending-results.json"))); } + @Test void writeAheadIntentRetainsItsUncommittedStateAcrossRestart() throws Exception { + UUID operationId = UUID.fromString("00000000-0000-0000-0000-000000000099"); + Route route = new Route("proxy-old", "Proxy Old", "VELOCITY", "7.1.2", + URI.create("https://control.example:8443"), "old-credential.txt", 30, 3000, 5000); + JsonObject result = new JsonObject(); + result.addProperty("revision", "anticipated-revision"); + + ProxyControlResultStore.save(directory, route, Map.of(operationId, new StoredResult(result, false))); + + assertFalse(ProxyControlResultStore.load(directory).results().get(operationId).committed()); + } + @Test void symbolicProxyResultJournalIsRejected() throws Exception { Path external = directory.resolve("external.json"); Files.writeString(external, "{}"); From 8f463832e8e68d4041531f0df8ddc4776e314081 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 20:28:06 -0600 Subject: [PATCH 66/73] Release results for forgotten operations --- .../control/BackendControlConnector.java | 46 ++++++++++++------- .../proxy/control/ControlConnector.java | 40 +++++++++++----- .../BackendControlConnectorProtocolTest.java | 4 ++ .../proxy/control/ControlConnectorTest.java | 23 ++++++++++ 4 files changed, 85 insertions(+), 28 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index 09347e4b79..ac2c3a3333 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -281,30 +281,44 @@ private boolean submitCompletedResult(UUID operationId, StoredResult submitted) persistCompleted(); return false; } + if (operationNotFound(response)) { + acknowledgeCompletedResult(operationId, submitted); + return true; + } afterResultAcknowledged( () -> requireObject(response, 200), - () -> { - synchronized (completed) { completed.remove(operationId); } - try { - persistCompleted(); - } catch (Exception failure) { - synchronized (completed) { completed.put(operationId, submitted); } - throw failure; - } - boolean drained; - synchronized (completed) { drained = completed.isEmpty(); } - if (drained && (submitted.restartConnector() || recovering)) { - plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); - } - }); + () -> acknowledgeCompletedResult(operationId, submitted)); return true; } + private void acknowledgeCompletedResult(UUID operationId, StoredResult submitted) throws Exception { + synchronized (completed) { completed.remove(operationId); } + try { + persistCompleted(); + } catch (Exception failure) { + synchronized (completed) { completed.put(operationId, submitted); } + throw failure; + } + boolean drained; + synchronized (completed) { drained = completed.isEmpty(); } + if (drained && (submitted.restartConnector() || recovering)) { + plugin.getServer().getScheduler().runTask(plugin, plugin::restartBackendControlConnector); + } + } + static boolean taskLeaseExpired(Response response) { - if (response.status() != 409) return false; + return responseCode(response, 409, "TASK_LEASE_EXPIRED"); + } + + static boolean operationNotFound(Response response) { + return responseCode(response, 404, "OPERATION_NOT_FOUND"); + } + + private static boolean responseCode(Response response, int status, String code) { + if (response.status() != status) return false; try { JsonObject error = JsonParser.parseString(response.body()).getAsJsonObject().getAsJsonObject("error"); - return error != null && error.has("code") && "TASK_LEASE_EXPIRED".equals(error.get("code").getAsString()); + return error != null && error.has("code") && code.equals(error.get("code").getAsString()); } catch (RuntimeException ignored) { return false; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index ff3a4662f5..3f5b1eecd0 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -477,25 +477,41 @@ private CompletableFuture submitCompletedResult(UUID operationId, StoredRe persistCompleted(); return; } - requireSuccess(resultResponse); - synchronized (operationLifecycle) { completedTasks.remove(operationId); } - try { - persistCompleted(); - } catch (RuntimeException failure) { - synchronized (operationLifecycle) { completedTasks.put(operationId, result); } - throw failure; + if (operationNotFound(resultResponse)) { + acknowledgeCompletedResult(operationId, result); + return; } - boolean drained; - synchronized (operationLifecycle) { drained = completedTasks.isEmpty(); } - if (recovering && drained && recoveryComplete != null) recoveryComplete.run(); + requireSuccess(resultResponse); + acknowledgeCompletedResult(operationId, result); }); } + private void acknowledgeCompletedResult(UUID operationId, StoredResult result) { + synchronized (operationLifecycle) { completedTasks.remove(operationId); } + try { + persistCompleted(); + } catch (RuntimeException failure) { + synchronized (operationLifecycle) { completedTasks.put(operationId, result); } + throw failure; + } + boolean drained; + synchronized (operationLifecycle) { drained = completedTasks.isEmpty(); } + if (recovering && drained && recoveryComplete != null) recoveryComplete.run(); + } + private static boolean taskLeaseExpired(Response response) { - if (response.statusCode != 409) return false; + return responseCode(response, 409, "TASK_LEASE_EXPIRED"); + } + + private static boolean operationNotFound(Response response) { + return responseCode(response, 404, "OPERATION_NOT_FOUND"); + } + + private static boolean responseCode(Response response, int status, String code) { + if (response.statusCode != status) return false; try { JsonObject error = parseObject(response.body).getAsJsonObject("error"); - return error != null && error.has("code") && "TASK_LEASE_EXPIRED".equals(error.get("code").getAsString()); + return error != null && error.has("code") && code.equals(error.get("code").getAsString()); } catch (RuntimeException ignored) { return false; } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index 7a30ef1105..9a2fd9b3fe 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -23,6 +23,10 @@ class BackendControlConnectorProtocolTest { assertFalse(BackendControlConnector.taskLeaseExpired(new BackendControlConnector.Response(409, "{\"error\":{\"code\":\"SESSION_MISMATCH\"}}"))); assertFalse(BackendControlConnector.taskLeaseExpired(new BackendControlConnector.Response(500, "not-json"))); + assertTrue(BackendControlConnector.operationNotFound(new BackendControlConnector.Response(404, + "{\"error\":{\"code\":\"OPERATION_NOT_FOUND\"}}"))); + assertFalse(BackendControlConnector.operationNotFound(new BackendControlConnector.Response(404, + "{\"error\":{\"code\":\"NODE_NOT_FOUND\"}}"))); } @Test void registrationRequiresFileControlButAllowsQuickSetupToRemainOptional() { diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index 7e77faf43e..e21092106a 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -280,6 +280,29 @@ class ControlConnectorTest { JsonParser.parseString(results.get(1).body()).getAsJsonObject().get("attemptId").getAsString()); } + @Test void forgottenControlOperationDoesNotBlockFutureClaims() { + connector.close(); + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L, + new ProxyRoutingConfigurationService(new NoOpPlatform())); + transport.acceptConfiguration = true; + transport.operationClaim = CompletableFuture.completedFuture(new Response(200, + "{\"operationId\":\"00000000-0000-0000-0000-000000000099\"," + + "\"attemptId\":\"00000000-0000-0000-0000-000000000199\"," + + "\"type\":\"READ\",\"configuration\":" + + "{\"sendVotesToAllServers\":false,\"blockedServers\":[]}}")); + transport.resultSubmission = CompletableFuture.completedFuture(new Response(404, + "{\"error\":{\"code\":\"OPERATION_NOT_FOUND\"}}")); + + connector.cycle(); + transport.operationClaim = CompletableFuture.completedFuture(new Response(204, "")); + connector.cycle(); + + assertEquals(2, transport.requests.stream().filter(request -> request.path().endsWith("/operations")).count()); + assertEquals(1, transport.requests.stream().filter(request -> request.path().endsWith("/result")).count()); + } + @Test void closeBeforeOperationPublicationPreventsTheRequestChainFromStarting() throws Exception { connector.close(); transport.firstSendEntered = new CountDownLatch(1); From cdf241c06a8eec7bf809e8b6f5e5588f3a69bdf2 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 20:40:03 -0600 Subject: [PATCH 67/73] Resolve abandoned configuration intents --- .../control/BackendControlConnector.java | 39 +++++++++++++-- .../control/BackendControlResultStore.java | 14 ++++-- .../proxy/control/ControlConnector.java | 47 +++++++++++++++++-- .../control/ProxyControlResultStore.java | 15 ++++-- .../BackendControlConnectorProtocolTest.java | 14 ++++++ .../BackendControlResultStoreTest.java | 10 ++-- .../proxy/control/ControlConnectorTest.java | 35 ++++++++++++++ .../control/ProxyControlResultStoreTest.java | 8 ++-- 8 files changed, 157 insertions(+), 25 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java index ac2c3a3333..77dd93662e 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlConnector.java @@ -238,7 +238,7 @@ private void claimAndExecute() throws Exception { result = completed.get(operationId); } if (result != null) { - if (!result.committed() && !anticipatedResultIsInstalled(result)) { + if (!result.committed() && !result.claimRequired() && !anticipatedResultIsInstalled(result)) { result = null; } else { result = committedForAttempt(result, string(task, "attemptId")); @@ -250,7 +250,7 @@ private void claimAndExecute() throws Exception { TaskResult executed = execute(operationId, task); JsonObject resultJson = executed.json(); resultJson.addProperty("attemptId", string(task, "attemptId")); - result = new StoredResult(resultJson, executed.restartConnector(), true); + result = new StoredResult(resultJson, executed.restartConnector(), true, false); synchronized (completed) { completed.put(operationId, result); } @@ -260,6 +260,7 @@ private void claimAndExecute() throws Exception { } private boolean submitCompletedResult() throws Exception { + prepareWriteAheadIntents(); Map.Entry pending; synchronized (completed) { pending = completed.entrySet().stream().filter(entry -> entry.getValue().committed()).findFirst().orElse(null); @@ -276,7 +277,8 @@ private boolean submitCompletedResult(UUID operationId, StoredResult submitted) + "/result", resultBody); if (taskLeaseExpired(response)) { synchronized (completed) { - completed.put(operationId, new StoredResult(submitted.result(), submitted.restartConnector(), false)); + completed.put(operationId, + new StoredResult(submitted.result(), submitted.restartConnector(), false, true)); } persistCompleted(); return false; @@ -334,11 +336,38 @@ private void persistIntent(UUID operationId, TaskResult anticipated, String atte JsonObject result = anticipated.json(); result.addProperty("attemptId", attemptId); synchronized (completed) { - completed.put(operationId, new StoredResult(result, anticipated.restartConnector(), false)); + completed.put(operationId, new StoredResult(result, anticipated.restartConnector(), false, false)); } persistCompleted(); } + private void prepareWriteAheadIntents() throws IOException { + Map snapshot; + synchronized (completed) { snapshot = new LinkedHashMap<>(completed); } + boolean changed = false; + for (Map.Entry entry : snapshot.entrySet()) { + StoredResult pending = entry.getValue(); + if (pending.committed() || pending.claimRequired()) continue; + StoredResult recovered = anticipatedResultIsInstalled(pending) + ? committedForAttempt(pending, string(pending.result(), "attemptId")) + : abortedIntent(pending); + synchronized (completed) { + if (completed.get(entry.getKey()) == pending) { + completed.put(entry.getKey(), recovered); + changed = true; + } + } + } + if (changed) persistCompleted(); + } + + static StoredResult abortedIntent(StoredResult pending) { + JsonObject result = TaskResult.failure("RECOVERY_ABORTED", + "Configuration apply did not finish before node recovery").json(); + result.addProperty("attemptId", string(pending.result(), "attemptId")); + return new StoredResult(result, false, true, false); + } + private boolean anticipatedResultIsInstalled(StoredResult pending) throws IOException { JsonObject result = pending.result(); if (!result.has("revision") || !result.has("configuration")) return false; @@ -357,7 +386,7 @@ private boolean anticipatedResultIsInstalled(StoredResult pending) throws IOExce private static StoredResult committedForAttempt(StoredResult pending, String attemptId) { JsonObject result = pending.result().deepCopy(); result.addProperty("attemptId", attemptId); - return new StoredResult(result, pending.restartConnector(), true); + return new StoredResult(result, pending.restartConnector(), true, false); } static void afterResultAcknowledged(ResultSubmission submission, ResultAcknowledgement acknowledged) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java index dfdc6dd82d..c105dba998 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java @@ -23,7 +23,7 @@ /** Durable result journal used to make locally applied Control operations restart-safe. */ final class BackendControlResultStore { - private static final int VERSION = 1; + private static final int VERSION = 2; // A valid 512 KiB YAML result may expand up to sixfold when characters require JSON unicode escaping. private static final int MAX_BYTES = 4 * 1024 * 1024; private static final int MAX_RESULTS = 128; @@ -66,10 +66,11 @@ static State load(Path dataDirectory) throws IOException { UUID operationId = UUID.fromString(string(item, "operationId")); JsonObject result = object(item, "result").deepCopy(); if (!item.has("restartConnector") || !item.get("restartConnector").isJsonPrimitive() - || !item.has("committed") || !item.get("committed").isJsonPrimitive()) throw invalid(); + || !item.has("committed") || !item.get("committed").isJsonPrimitive() + || !item.has("claimRequired") || !item.get("claimRequired").isJsonPrimitive()) throw invalid(); StoredResult previous = results.put(operationId, new StoredResult(result, item.get("restartConnector").getAsBoolean(), - item.get("committed").getAsBoolean())); + item.get("committed").getAsBoolean(), item.get("claimRequired").getAsBoolean())); if (previous != null) throw invalid(); } return new State(route, Map.copyOf(results)); @@ -107,6 +108,7 @@ static void save(Path dataDirectory, Route route, Map result item.add("result", result.result().deepCopy()); item.addProperty("restartConnector", result.restartConnector()); item.addProperty("committed", result.committed()); + item.addProperty("claimRequired", result.claimRequired()); listed.add(item); }); root.add("results", listed); @@ -158,6 +160,10 @@ private static IllegalArgumentException invalid() { record Route(String nodeId, URI endpoint, String credentialFile, int heartbeatSeconds, int connectTimeoutMillis, int requestTimeoutMillis) { } - record StoredResult(JsonObject result, boolean restartConnector, boolean committed) { } + record StoredResult(JsonObject result, boolean restartConnector, boolean committed, boolean claimRequired) { + StoredResult { + if (committed && claimRequired) throw new IllegalArgumentException("committed result cannot require a claim"); + } + } record State(Route route, Map results) { } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java index 3f5b1eecd0..8ee78d1778 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ControlConnector.java @@ -96,6 +96,15 @@ public ControlConnector(Settings settings, ScheduledExecutorService scheduler, T null, null, false, null); } + ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, + Supplier> snapshotSource, Consumer logger, UUID sessionId, + LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService, + Map recoveredTasks) { + this(settings, scheduler, transport, snapshotSource, logger, sessionId, jitterSource, configurationService, + null, null, false, null); + completedTasks.putAll(recoveredTasks); + } + private ControlConnector(Settings settings, ScheduledExecutorService scheduler, Transport transport, Supplier> snapshotSource, Consumer logger, UUID sessionId, LongSupplier jitterSource, ProxyRoutingConfigurationService configurationService, Path dataDirectory, @@ -428,7 +437,7 @@ private CompletableFuture handleClaimResponse(Response response) { result = completedTasks.get(operationId); } if (result != null) { - if (!result.committed() && !anticipatedResultIsInstalled(result)) { + if (!result.committed() && !result.claimRequired() && !anticipatedResultIsInstalled(result)) { result = null; } else { result = committedForAttempt(result, requireString(task, "attemptId")); @@ -440,7 +449,7 @@ private CompletableFuture handleClaimResponse(Response response) { TaskResult executed = executeTask(operationId, task); JsonObject resultJson = executed.json(); resultJson.addProperty("attemptId", requireString(task, "attemptId")); - result = new StoredResult(resultJson, true); + result = new StoredResult(resultJson, true, false); synchronized (operationLifecycle) { completedTasks.put(operationId, result); } @@ -450,6 +459,7 @@ private CompletableFuture handleClaimResponse(Response response) { } private boolean hasCompletedTask() { + prepareWriteAheadIntents(); synchronized (operationLifecycle) { return completedTasks.values().stream().anyMatch(StoredResult::committed); } @@ -472,7 +482,7 @@ private CompletableFuture submitCompletedResult(UUID operationId, StoredRe return submitted.thenAccept(resultResponse -> { if (taskLeaseExpired(resultResponse)) { synchronized (operationLifecycle) { - completedTasks.put(operationId, new StoredResult(result.result(), false)); + completedTasks.put(operationId, new StoredResult(result.result(), false, true)); } persistCompleted(); return; @@ -532,11 +542,38 @@ private void persistIntent(UUID operationId, TaskResult anticipated, String atte JsonObject result = anticipated.json(); result.addProperty("attemptId", attemptId); synchronized (operationLifecycle) { - completedTasks.put(operationId, new StoredResult(result, false)); + completedTasks.put(operationId, new StoredResult(result, false, false)); } persistCompleted(); } + private void prepareWriteAheadIntents() { + Map snapshot; + synchronized (operationLifecycle) { snapshot = new LinkedHashMap<>(completedTasks); } + boolean changed = false; + for (Map.Entry entry : snapshot.entrySet()) { + StoredResult pending = entry.getValue(); + if (pending.committed() || pending.claimRequired()) continue; + StoredResult recovered = anticipatedResultIsInstalled(pending) + ? committedForAttempt(pending, requireString(pending.result(), "attemptId")) + : abortedIntent(pending); + synchronized (operationLifecycle) { + if (completedTasks.get(entry.getKey()) == pending) { + completedTasks.put(entry.getKey(), recovered); + changed = true; + } + } + } + if (changed) persistCompleted(); + } + + private static StoredResult abortedIntent(StoredResult pending) { + JsonObject result = TaskResult.failure("RECOVERY_ABORTED", + "Configuration apply did not finish before node recovery").json(); + result.addProperty("attemptId", requireString(pending.result(), "attemptId")); + return new StoredResult(result, true, false); + } + private boolean anticipatedResultIsInstalled(StoredResult pending) { JsonObject result = pending.result(); return result.has("revision") && result.get("revision").getAsString().equals(configurationService.read().revision()); @@ -545,7 +582,7 @@ private boolean anticipatedResultIsInstalled(StoredResult pending) { private static StoredResult committedForAttempt(StoredResult pending, String attemptId) { JsonObject result = pending.result().deepCopy(); result.addProperty("attemptId", attemptId); - return new StoredResult(result, true); + return new StoredResult(result, true, false); } private TaskResult executeTask(UUID operationId, JsonObject task) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java index 6bce1705c7..bb34c50f34 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java @@ -23,7 +23,7 @@ /** Durable result journal for proxy-routing operations accepted by a specific Control node. */ final class ProxyControlResultStore { - private static final int VERSION = 1; + private static final int VERSION = 2; private static final int MAX_BYTES = 256 * 1024; private static final int MAX_RESULTS = 128; private static final String FILE_NAME = ".control-proxy-pending-results.json"; @@ -65,9 +65,11 @@ static State load(Path dataDirectory) throws IOException { if (!element.isJsonObject()) throw invalid(); JsonObject item = element.getAsJsonObject(); UUID operationId = UUID.fromString(string(item, "operationId")); - if (!item.has("committed") || !item.get("committed").isJsonPrimitive()) throw invalid(); + if (!item.has("committed") || !item.get("committed").isJsonPrimitive() + || !item.has("claimRequired") || !item.get("claimRequired").isJsonPrimitive()) throw invalid(); StoredResult previous = results.put(operationId, - new StoredResult(object(item, "result").deepCopy(), item.get("committed").getAsBoolean())); + new StoredResult(object(item, "result").deepCopy(), item.get("committed").getAsBoolean(), + item.get("claimRequired").getAsBoolean())); if (previous != null) throw invalid(); } return new State(route, Map.copyOf(results)); @@ -107,6 +109,7 @@ static void save(Path dataDirectory, Route route, Map result item.addProperty("operationId", operationId.toString()); item.add("result", result.result().deepCopy()); item.addProperty("committed", result.committed()); + item.addProperty("claimRequired", result.claimRequired()); listed.add(item); }); root.add("results", listed); @@ -158,6 +161,10 @@ private static IllegalArgumentException invalid() { record Route(String nodeId, String displayName, String platform, String pluginVersion, URI endpoint, String credentialFile, int heartbeatSeconds, int connectTimeoutMillis, int requestTimeoutMillis) { } - record StoredResult(JsonObject result, boolean committed) { } + record StoredResult(JsonObject result, boolean committed, boolean claimRequired) { + StoredResult { + if (committed && claimRequired) throw new IllegalArgumentException("committed result cannot require a claim"); + } + } record State(Route route, Map results) { } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java index 9a2fd9b3fe..2be4bd6656 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlConnectorProtocolTest.java @@ -13,6 +13,7 @@ import org.junit.jupiter.api.Test; +import com.bencodez.votingplugin.control.BackendControlResultStore.StoredResult; import com.google.gson.JsonArray; import com.google.gson.JsonObject; @@ -29,6 +30,19 @@ class BackendControlConnectorProtocolTest { "{\"error\":{\"code\":\"NODE_NOT_FOUND\"}}"))); } + @Test void abandonedBackendIntentBecomesATerminalRecoveryResult() { + JsonObject anticipated = new JsonObject(); + anticipated.addProperty("attemptId", "00000000-0000-0000-0000-000000000199"); + + StoredResult recovered = BackendControlConnector.abortedIntent( + new StoredResult(anticipated, true, false, false)); + + assertTrue(recovered.committed()); + assertFalse(recovered.claimRequired()); + assertFalse(recovered.restartConnector()); + assertTrue("RECOVERY_ABORTED".equals(recovered.result().get("code").getAsString())); + } + @Test void registrationRequiresFileControlButAllowsQuickSetupToRemainOptional() { assertThrows(RuntimeException.class, () -> BackendControlConnector.requireFileCapability(false)); assertDoesNotThrow(() -> BackendControlConnector.requireFileCapability(true)); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java index 6b53217c99..bdb10fde88 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendControlResultStoreTest.java @@ -30,7 +30,7 @@ class BackendControlResultStoreTest { result.addProperty("success", true); result.addProperty("revision", "applied-revision"); Map pending = new LinkedHashMap<>(); - pending.put(operationId, new StoredResult(result, true, true)); + pending.put(operationId, new StoredResult(result, true, true, false)); BackendControlResultStore.save(directory, route, pending); BackendControlResultStore.State recovered = BackendControlResultStore.load(directory); @@ -50,9 +50,11 @@ class BackendControlResultStoreTest { result.addProperty("revision", "anticipated-revision"); BackendControlResultStore.save(directory, route, - Map.of(operationId, new StoredResult(result, false, false))); + Map.of(operationId, new StoredResult(result, false, false, false))); - assertFalse(BackendControlResultStore.load(directory).results().get(operationId).committed()); + StoredResult recovered = BackendControlResultStore.load(directory).results().get(operationId); + assertFalse(recovered.committed()); + assertFalse(recovered.claimRequired()); } @Test void symbolicPendingResultJournalIsRejected() throws Exception { @@ -76,7 +78,7 @@ class BackendControlResultStoreTest { BackendControlResultStore.save(directory, route, Map.of(UUID.fromString("00000000-0000-0000-0000-000000000099"), - new StoredResult(result, false, true))); + new StoredResult(result, false, true, false))); assertEquals(BackendConfigurationService.MAX_CONTENT_BYTES, BackendControlResultStore.load(directory).results().values().iterator().next().result() diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java index e21092106a..8fe3dfdc05 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ControlConnectorTest.java @@ -6,11 +6,13 @@ import com.bencodez.votingplugin.proxy.control.ControlConnector.Settings; import com.bencodez.votingplugin.proxy.control.ControlConnector.Status; import com.bencodez.votingplugin.proxy.control.ControlConnector.Transport; +import com.bencodez.votingplugin.proxy.control.ProxyControlResultStore.StoredResult; import com.google.gson.JsonObject; import com.google.gson.JsonParser; import java.net.URI; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CountDownLatch; @@ -303,6 +305,39 @@ class ControlConnectorTest { assertEquals(1, transport.requests.stream().filter(request -> request.path().endsWith("/result")).count()); } + @Test void abandonedWriteAheadIntentIsReportedAndReleasedWhenControlForgotTheOperation() { + connector.close(); + UUID operationId = UUID.fromString("00000000-0000-0000-0000-000000000099"); + ProxyRoutingConfiguration proposal = new ProxyRoutingConfiguration(true, List.of()); + JsonObject anticipated = new JsonObject(); + anticipated.addProperty("success", true); + anticipated.addProperty("code", "OK"); + anticipated.addProperty("message", "Configuration applied"); + anticipated.addProperty("revision", proposal.revision()); + JsonObject configuration = new JsonObject(); + configuration.addProperty("sendVotesToAllServers", true); + configuration.add("blockedServers", new com.google.gson.JsonArray()); + anticipated.add("configuration", configuration); + anticipated.addProperty("attemptId", "00000000-0000-0000-0000-000000000199"); + connector = new ControlConnector(settings(), scheduler, transport, + () -> List.of(), logs::add, UUID.randomUUID(), () -> 0L, + new ProxyRoutingConfigurationService(new NoOpPlatform()), + Map.of(operationId, new StoredResult(anticipated, false, false))); + transport.acceptConfiguration = true; + transport.resultSubmission = CompletableFuture.completedFuture(new Response(404, + "{\"error\":{\"code\":\"OPERATION_NOT_FOUND\"}}")); + + connector.cycle(); + JsonObject submitted = JsonParser.parseString(transport.requests.stream() + .filter(request -> request.path().endsWith("/result")).findFirst().orElseThrow().body()).getAsJsonObject(); + assertEquals("RECOVERY_ABORTED", submitted.get("code").getAsString()); + + transport.operationClaim = CompletableFuture.completedFuture(new Response(204, "")); + connector.cycle(); + assertEquals(1, transport.requests.stream().filter(request -> request.path().endsWith("/operations")).count()); + assertTrue(connector.reserveRuntimeReplacement()); + } + @Test void closeBeforeOperationPublicationPreventsTheRequestChainFromStarting() throws Exception { connector.close(); transport.firstSendEntered = new CountDownLatch(1); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java index e6bdddfad9..5f6551bde0 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStoreTest.java @@ -30,7 +30,7 @@ class ProxyControlResultStoreTest { result.addProperty("success", true); result.addProperty("revision", "applied-revision"); Map pending = new LinkedHashMap<>(); - pending.put(operationId, new StoredResult(result, true)); + pending.put(operationId, new StoredResult(result, true, false)); ProxyControlResultStore.save(directory, route, pending); ProxyControlResultStore.State recovered = ProxyControlResultStore.load(directory); @@ -49,9 +49,11 @@ class ProxyControlResultStoreTest { JsonObject result = new JsonObject(); result.addProperty("revision", "anticipated-revision"); - ProxyControlResultStore.save(directory, route, Map.of(operationId, new StoredResult(result, false))); + ProxyControlResultStore.save(directory, route, Map.of(operationId, new StoredResult(result, false, false))); - assertFalse(ProxyControlResultStore.load(directory).results().get(operationId).committed()); + StoredResult recovered = ProxyControlResultStore.load(directory).results().get(operationId); + assertFalse(recovered.committed()); + assertFalse(recovered.claimRequired()); } @Test void symbolicProxyResultJournalIsRejected() throws Exception { From 1ec4fbff20786ef09b5bbcf75c7a02255e0c41ce Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 20:57:58 -0600 Subject: [PATCH 68/73] Require atomic result-journal publication --- .../votingplugin/control/BackendControlResultStore.java | 7 +------ .../proxy/control/ProxyControlResultStore.java | 7 +------ 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java index c105dba998..9bd9ba4d68 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java @@ -5,7 +5,6 @@ import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; -import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; @@ -118,11 +117,7 @@ static void save(Path dataDirectory, Route route, Map result Path staging = Files.createTempFile(dataDirectory, ".control-results-", ".json"); try { Files.write(staging, bytes, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); - try { - Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (AtomicMoveNotSupportedException e) { - Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING); - } + Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } finally { Files.deleteIfExists(staging); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java index bb34c50f34..4960df38d7 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java @@ -5,7 +5,6 @@ import java.nio.ByteBuffer; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; -import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; @@ -119,11 +118,7 @@ static void save(Path dataDirectory, Route route, Map result Path staging = Files.createTempFile(dataDirectory, ".control-proxy-results-", ".json"); try { Files.write(staging, bytes, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); - try { - Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - } catch (AtomicMoveNotSupportedException e) { - Files.move(staging, target, StandardCopyOption.REPLACE_EXISTING); - } + Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); } finally { Files.deleteIfExists(staging); } From f83a6e97bdc0f151ededb9c7207dd3a1968d2adb Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 21:14:29 -0600 Subject: [PATCH 69/73] Bound Redis handoff and durably publish result journals --- .../cache/ProcessedVoteCache.java | 12 +++++++-- .../transport/RedisBackendProxyTransport.java | 14 ++++++++-- .../control/BackendControlResultStore.java | 17 ++++++++++-- .../control/ProxyControlResultStore.java | 17 ++++++++++-- .../RedisBackendProxyTransportTest.java | 26 +++++++++++++++++++ .../cache/ProcessedVoteCacheTest.java | 19 ++++++++++++++ 6 files changed, 97 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index e91e2914f0..7467b699f1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -1,9 +1,9 @@ package com.bencodez.votingplugin.backendproxy.cache; -import java.util.UUID; import java.util.Iterator; import java.util.LinkedHashMap; import java.util.Map; +import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; @@ -91,7 +91,15 @@ public synchronized void registerRedisSubscriber(Object subscriber) { /** Returns true only for the active subscriber and counts its legacy delivery during overlap. */ public synchronized boolean reserveLegacyRedisDelivery(Object subscriber, String signature) { if (activeRedisSubscriber != subscriber) return false; - if (standbyRedisSubscriber != null) legacyRedisDeliveries.merge(signature, 1, Integer::sum); + if (standbyRedisSubscriber != null) { + Integer count = legacyRedisDeliveries.get(signature); + if (count != null) { + legacyRedisDeliveries.put(signature, + count == Integer.MAX_VALUE ? Integer.MAX_VALUE : count + 1); + } else if (legacyRedisDeliveries.size() < MAX_REDIS_DELIVERIES) { + legacyRedisDeliveries.put(signature, 1); + } + } return true; } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index ce0d56a959..33003d50b6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -22,6 +22,7 @@ import lombok.Getter; public class RedisBackendProxyTransport implements BackendProxyTransport { + static final int MAX_LEGACY_HANDOFF_DELIVERIES = 4096; private final VotingPluginMain plugin; private final ProcessedVoteCache processedVoteCache; @@ -32,6 +33,7 @@ public class RedisBackendProxyTransport implements BackendProxyTransport { private final Object subscriberIdentity = new Object(); private final Object legacyLifecycle = new Object(); private final List bufferedLegacyDeliveries = new ArrayList<>(); + private boolean legacyHandoffDegraded; private GlobalMessageHandler messageHandler; public RedisBackendProxyTransport(VotingPluginMain plugin) { @@ -86,13 +88,20 @@ public void onSubscribe(String channel, int subscribedChannels) { listenerThread.start(); } - private void dispatchLegacy(JsonEnvelope envelope) { + void dispatchLegacy(JsonEnvelope envelope) { String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); synchronized (legacyLifecycle) { if (processedVoteCache.reserveLegacyRedisDelivery(subscriberIdentity, signature)) { messageHandler.onMessage(envelope); - } else { + } else if (bufferedLegacyDeliveries.size() < MAX_LEGACY_HANDOFF_DELIVERIES) { bufferedLegacyDeliveries.add(envelope); + } else { + if (!legacyHandoffDegraded && plugin != null) { + plugin.getLogger().warning("Redis legacy handoff exceeded " + MAX_LEGACY_HANDOFF_DELIVERIES + + " buffered deliveries; temporarily degrading duplicate suppression"); + } + legacyHandoffDegraded = true; + messageHandler.onMessage(envelope); } } } @@ -106,6 +115,7 @@ public void activateAfterHandoff() { if (!processedVoteCache.consumeLegacyRedisDelivery(signature)) messageHandler.onMessage(envelope); } bufferedLegacyDeliveries.clear(); + legacyHandoffDegraded = false; processedVoteCache.finishRedisHandoff(); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java index 9bd9ba4d68..8acca4f77f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -82,7 +83,7 @@ static void save(Path dataDirectory, Route route, Map result Path target = target(dataDirectory); if (results.isEmpty()) { if (Files.isSymbolicLink(target)) throw new IOException("Control pending-result journal is unsafe"); - Files.deleteIfExists(target); + if (Files.deleteIfExists(target)) forceDirectory(dataDirectory); return; } if (results.size() > MAX_RESULTS) throw new IOException("Too many pending Control results"); @@ -116,13 +117,25 @@ static void save(Path dataDirectory, Route route, Map result Files.createDirectories(dataDirectory); Path staging = Files.createTempFile(dataDirectory, ".control-results-", ".json"); try { - Files.write(staging, bytes, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + try (FileChannel channel = FileChannel.open(staging, StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE)) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) channel.write(buffer); + channel.force(true); + } Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + forceDirectory(dataDirectory); } finally { Files.deleteIfExists(staging); } } + private static void forceDirectory(Path directory) throws IOException { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } + private static Path target(Path dataDirectory) { return dataDirectory.toAbsolutePath().normalize().resolve(FILE_NAME); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java index 4960df38d7..e8ba216e77 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.URI; import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -81,7 +82,7 @@ static void save(Path dataDirectory, Route route, Map result Path target = target(dataDirectory); if (results.isEmpty()) { if (Files.isSymbolicLink(target)) throw new IOException("Control proxy-result journal is unsafe"); - Files.deleteIfExists(target); + if (Files.deleteIfExists(target)) forceDirectory(dataDirectory); return; } if (results.size() > MAX_RESULTS) throw new IOException("Too many pending Control proxy results"); @@ -117,13 +118,25 @@ static void save(Path dataDirectory, Route route, Map result Files.createDirectories(dataDirectory); Path staging = Files.createTempFile(dataDirectory, ".control-proxy-results-", ".json"); try { - Files.write(staging, bytes, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); + try (FileChannel channel = FileChannel.open(staging, StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE)) { + ByteBuffer buffer = ByteBuffer.wrap(bytes); + while (buffer.hasRemaining()) channel.write(buffer); + channel.force(true); + } Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + forceDirectory(dataDirectory); } finally { Files.deleteIfExists(staging); } } + private static void forceDirectory(Path directory) throws IOException { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } + private static Path target(Path dataDirectory) { return dataDirectory.toAbsolutePath().normalize().resolve(FILE_NAME); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index af46ee44f5..de7a4ccb97 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -3,9 +3,19 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +import java.lang.reflect.Field; import org.junit.jupiter.api.Test; +import com.bencodez.simpleapi.servercomm.codec.JsonEnvelope; +import com.bencodez.simpleapi.servercomm.global.GlobalMessageHandler; +import com.bencodez.votingplugin.backendproxy.cache.ProcessedVoteCache; +import com.bencodez.votingplugin.proxy.VotingPluginWire; + import redis.clients.jedis.DefaultJedisClientConfig; class RedisBackendProxyTransportTest { @@ -24,4 +34,20 @@ void validationHonorsTlsAndHostnameVerification() { void validationKeepsTlsDisabledByDefault() { assertFalse(RedisBackendProxyTransport.buildValidationClientConfig(0, null, null, false).isSsl()); } + + @Test + void legacyHandoffBufferDegradesAtItsFixedLimit() throws Exception { + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, new ProcessedVoteCache()); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + Field handler = RedisBackendProxyTransport.class.getDeclaredField("messageHandler"); + handler.setAccessible(true); + handler.set(transport, messages); + JsonEnvelope envelope = JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE).build(); + + for (int index = 0; index <= RedisBackendProxyTransport.MAX_LEGACY_HANDOFF_DELIVERIES; index++) { + transport.dispatchLegacy(envelope); + } + + verify(messages, times(1)).onMessage(envelope); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java index fe54a6184d..a7e23b9be8 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java @@ -4,6 +4,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.lang.reflect.Field; +import java.util.Map; import java.util.UUID; import org.junit.jupiter.api.Test; @@ -64,4 +66,21 @@ public void legacyRedisHandoffMatchesOldAndBufferedDeliveryCounts() { assertFalse(cache.consumeLegacyRedisDelivery("same-envelope")); cache.finishRedisHandoff(); } + + @Test + public void legacyRedisHandoffAccountingIsBounded() throws Exception { + ProcessedVoteCache cache = new ProcessedVoteCache(); + Object previous = new Object(); + Object replacement = new Object(); + cache.registerRedisSubscriber(previous); + cache.registerRedisSubscriber(replacement); + + for (int index = 0; index < 5000; index++) { + assertTrue(cache.reserveLegacyRedisDelivery(previous, "envelope-" + index)); + } + + Field deliveries = ProcessedVoteCache.class.getDeclaredField("legacyRedisDeliveries"); + deliveries.setAccessible(true); + assertEquals(4096, ((Map) deliveries.get(cache)).size()); + } } From 79f928642a3e951b54452733e46d9edbad8c8318 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 21:22:57 -0600 Subject: [PATCH 70/73] Keep durable journals portable on Windows --- .../control/BackendControlResultStore.java | 17 +++++++++++++++-- .../proxy/control/ProxyControlResultStore.java | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java index 8acca4f77f..7e2745bc97 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java @@ -6,12 +6,14 @@ import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -131,8 +133,19 @@ static void save(Path dataDirectory, Route route, Map result } private static void forceDirectory(Path directory) throws IOException { - try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { - channel.force(true); + try { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } catch (AccessDeniedException unsupportedDirectoryHandle) { + // The Windows NIO provider cannot open directory handles. The staged file was + // already forced and the atomic move has completed, so do not turn every + // otherwise successful journal update into a permanent Windows failure. + if (!System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win")) { + throw unsupportedDirectoryHandle; + } + } catch (UnsupportedOperationException unsupportedDirectoryForce) { + // Some providers support atomic moves but expose no directory-force operation. } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java index e8ba216e77..6999de42f9 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java @@ -6,12 +6,14 @@ import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; +import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.LinkedHashMap; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; @@ -132,8 +134,19 @@ static void save(Path dataDirectory, Route route, Map result } private static void forceDirectory(Path directory) throws IOException { - try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { - channel.force(true); + try { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } catch (AccessDeniedException unsupportedDirectoryHandle) { + // The Windows NIO provider cannot open directory handles. The staged file was + // already forced and the atomic move has completed, so do not turn every + // otherwise successful journal update into a permanent Windows failure. + if (!System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win")) { + throw unsupportedDirectoryHandle; + } + } catch (UnsupportedOperationException unsupportedDirectoryForce) { + // Some providers support atomic moves but expose no directory-force operation. } } From ccbba461b8c4f14c957f90125eed05bd89db5127 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 21:35:41 -0600 Subject: [PATCH 71/73] Durably publish configuration and hosted state --- .../cache/ProcessedVoteCache.java | 40 ++++++++++++-- .../transport/RedisBackendProxyTransport.java | 17 ++++-- .../control/BackendConfigurationService.java | 3 ++ .../control/BackendControlResultStore.java | 24 ++------- .../proxy/bungee/BungeeConfig.java | 3 ++ .../proxy/control/HostedControlManager.java | 13 +++-- .../control/ProxyControlResultStore.java | 24 ++------- .../proxy/velocity/VelocityConfig.java | 3 ++ .../votingplugin/util/DurableFiles.java | 53 +++++++++++++++++++ .../RedisBackendProxyTransportTest.java | 15 ++++++ .../cache/ProcessedVoteCacheTest.java | 15 ++++++ .../votingplugin/util/DurableFilesTest.java | 16 ++++++ 12 files changed, 170 insertions(+), 56 deletions(-) create mode 100644 VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/util/DurableFilesTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java index 7467b699f1..919fb8077d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/cache/ProcessedVoteCache.java @@ -16,12 +16,15 @@ public class ProcessedVoteCache { private static final long DEFAULT_TTL_MILLIS = TimeUnit.MINUTES.toMillis(30); private static final int MAX_REDIS_DELIVERIES = 4096; + public static final int MAX_LEGACY_REDIS_DELIVERY_BYTES = 256 * 1024; + public static final int MAX_LEGACY_REDIS_TOTAL_BYTES = 4 * 1024 * 1024; @Getter private final ConcurrentHashMap processedVotes = new ConcurrentHashMap<>(); private final long ttlMillis; private final LinkedHashMap processedRedisDeliveries = new LinkedHashMap<>(); private final LinkedHashMap legacyRedisDeliveries = new LinkedHashMap<>(); + private long legacyRedisDeliveryBytes; private Object activeRedisSubscriber; private Object standbyRedisSubscriber; @@ -85,6 +88,7 @@ public synchronized void registerRedisSubscriber(Object subscriber) { } else if (activeRedisSubscriber != subscriber) { standbyRedisSubscriber = subscriber; legacyRedisDeliveries.clear(); + legacyRedisDeliveryBytes = 0; } } @@ -96,8 +100,14 @@ public synchronized boolean reserveLegacyRedisDelivery(Object subscriber, String if (count != null) { legacyRedisDeliveries.put(signature, count == Integer.MAX_VALUE ? Integer.MAX_VALUE : count + 1); - } else if (legacyRedisDeliveries.size() < MAX_REDIS_DELIVERIES) { - legacyRedisDeliveries.put(signature, 1); + } else { + int bytes = legacyRedisDeliveryBytes(signature); + if (bytes <= MAX_LEGACY_REDIS_DELIVERY_BYTES + && legacyRedisDeliveries.size() < MAX_REDIS_DELIVERIES + && legacyRedisDeliveryBytes <= MAX_LEGACY_REDIS_TOTAL_BYTES - bytes) { + legacyRedisDeliveries.put(signature, 1); + legacyRedisDeliveryBytes += bytes; + } } } return true; @@ -115,19 +125,23 @@ public synchronized void activateRedisSubscriber(Object subscriber) { public synchronized boolean consumeLegacyRedisDelivery(String signature) { Integer count = legacyRedisDeliveries.get(signature); if (count == null) return false; - if (count <= 1) legacyRedisDeliveries.remove(signature); - else legacyRedisDeliveries.put(signature, count - 1); + if (count <= 1) { + legacyRedisDeliveries.remove(signature); + legacyRedisDeliveryBytes -= legacyRedisDeliveryBytes(signature); + } else legacyRedisDeliveries.put(signature, count - 1); return true; } public synchronized void finishRedisHandoff() { legacyRedisDeliveries.clear(); + legacyRedisDeliveryBytes = 0; } public synchronized void unregisterRedisSubscriber(Object subscriber) { if (standbyRedisSubscriber == subscriber) { standbyRedisSubscriber = null; legacyRedisDeliveries.clear(); + legacyRedisDeliveryBytes = 0; } if (activeRedisSubscriber == subscriber) activeRedisSubscriber = null; } @@ -135,4 +149,22 @@ public synchronized void unregisterRedisSubscriber(Object subscriber) { private void cleanup(long now) { processedVotes.entrySet().removeIf(entry -> entry.getValue() <= now); } + + /** Returns an exact UTF-8 length up to the per-delivery cap, then cap + 1. */ + public static int legacyRedisDeliveryBytes(String signature) { + if (signature == null) return 0; + int bytes = 0; + for (int index = 0; index < signature.length(); index++) { + char character = signature.charAt(index); + if (character <= 0x7f) bytes++; + else if (character <= 0x7ff) bytes += 2; + else if (Character.isHighSurrogate(character) && index + 1 < signature.length() + && Character.isLowSurrogate(signature.charAt(index + 1))) { + bytes += 4; + index++; + } else bytes += 3; + if (bytes > MAX_LEGACY_REDIS_DELIVERY_BYTES) return MAX_LEGACY_REDIS_DELIVERY_BYTES + 1; + } + return bytes; + } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java index 33003d50b6..eb2ce25391 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransport.java @@ -1,9 +1,9 @@ package com.bencodez.votingplugin.backendproxy.transport; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; import javax.net.ssl.SSLParameters; @@ -33,6 +33,7 @@ public class RedisBackendProxyTransport implements BackendProxyTransport { private final Object subscriberIdentity = new Object(); private final Object legacyLifecycle = new Object(); private final List bufferedLegacyDeliveries = new ArrayList<>(); + private long bufferedLegacyDeliveryBytes; private boolean legacyHandoffDegraded; private GlobalMessageHandler messageHandler; @@ -90,15 +91,20 @@ public void onSubscribe(String channel, int subscribedChannels) { void dispatchLegacy(JsonEnvelope envelope) { String signature = com.bencodez.simpleapi.servercomm.codec.JsonEnvelopeCodec.encode(envelope); + int encodedBytes = ProcessedVoteCache.legacyRedisDeliveryBytes(signature); synchronized (legacyLifecycle) { if (processedVoteCache.reserveLegacyRedisDelivery(subscriberIdentity, signature)) { messageHandler.onMessage(envelope); - } else if (bufferedLegacyDeliveries.size() < MAX_LEGACY_HANDOFF_DELIVERIES) { + } else if (encodedBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES + && bufferedLegacyDeliveries.size() < MAX_LEGACY_HANDOFF_DELIVERIES + && bufferedLegacyDeliveryBytes <= ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES - encodedBytes) { bufferedLegacyDeliveries.add(envelope); + bufferedLegacyDeliveryBytes += encodedBytes; } else { if (!legacyHandoffDegraded && plugin != null) { - plugin.getLogger().warning("Redis legacy handoff exceeded " + MAX_LEGACY_HANDOFF_DELIVERIES - + " buffered deliveries; temporarily degrading duplicate suppression"); + plugin.getLogger().warning("Redis legacy handoff exceeded its " + MAX_LEGACY_HANDOFF_DELIVERIES + + " delivery / " + ProcessedVoteCache.MAX_LEGACY_REDIS_TOTAL_BYTES + + " byte buffer; temporarily degrading duplicate suppression"); } legacyHandoffDegraded = true; messageHandler.onMessage(envelope); @@ -115,6 +121,7 @@ public void activateAfterHandoff() { if (!processedVoteCache.consumeLegacyRedisDelivery(signature)) messageHandler.onMessage(envelope); } bufferedLegacyDeliveries.clear(); + bufferedLegacyDeliveryBytes = 0; legacyHandoffDegraded = false; processedVoteCache.finishRedisHandoff(); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index f50fa05325..dbee823003 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -27,6 +27,7 @@ import org.bukkit.configuration.file.YamlConfiguration; import com.bencodez.votingplugin.proxy.BungeeMethod; +import com.bencodez.votingplugin.util.DurableFiles; /** Safe, revisioned access to every user-facing VotingPlugin YAML configuration file. */ public final class BackendConfigurationService { @@ -368,7 +369,9 @@ private static String revision(String content) { } private static void move(Path source, Path target) throws IOException { + DurableFiles.forceFile(source); Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + DurableFiles.forceMoveDirectories(source, target); } private static String bungeeMethodOption(Map options) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java index 7e2745bc97..7d9ce39dc5 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendControlResultStore.java @@ -6,18 +6,17 @@ import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; -import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.LinkedHashMap; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; +import com.bencodez.votingplugin.util.DurableFiles; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -85,7 +84,7 @@ static void save(Path dataDirectory, Route route, Map result Path target = target(dataDirectory); if (results.isEmpty()) { if (Files.isSymbolicLink(target)) throw new IOException("Control pending-result journal is unsafe"); - if (Files.deleteIfExists(target)) forceDirectory(dataDirectory); + DurableFiles.deleteIfExists(target); return; } if (results.size() > MAX_RESULTS) throw new IOException("Too many pending Control results"); @@ -126,29 +125,12 @@ static void save(Path dataDirectory, Route route, Map result channel.force(true); } Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - forceDirectory(dataDirectory); + DurableFiles.forceDirectory(dataDirectory); } finally { Files.deleteIfExists(staging); } } - private static void forceDirectory(Path directory) throws IOException { - try { - try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { - channel.force(true); - } - } catch (AccessDeniedException unsupportedDirectoryHandle) { - // The Windows NIO provider cannot open directory handles. The staged file was - // already forced and the atomic move has completed, so do not turn every - // otherwise successful journal update into a permanent Windows failure. - if (!System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win")) { - throw unsupportedDirectoryHandle; - } - } catch (UnsupportedOperationException unsupportedDirectoryForce) { - // Some providers support atomic moves but expose no directory-force operation. - } - } - private static Path target(Path dataDirectory) { return dataDirectory.toAbsolutePath().normalize().resolve(FILE_NAME); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index 9f7859acba..5e0ac585d6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -19,6 +19,7 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.control.ProxyRoutingConfiguration; +import com.bencodez.votingplugin.util.DurableFiles; import lombok.Getter; import net.md_5.bungee.config.Configuration; @@ -625,7 +626,9 @@ public synchronized void verifyControlProxyRoutingSnapshot(byte[] snapshot) thro private static void atomicReplace(Path source, Path target) throws IOException { try { + DurableFiles.forceFile(source); Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + DurableFiles.forceMoveDirectories(source, target); } catch (java.nio.file.AtomicMoveNotSupportedException e) { throw new IOException("Atomic Control configuration activation is unsupported", e); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java index 0f8e8fb9aa..528a744c3c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/HostedControlManager.java @@ -36,6 +36,7 @@ import com.bencodez.votingplugin.proxy.VotingPluginProxy; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.util.BoundedHttpBodyHandler; +import com.bencodez.votingplugin.util.DurableFiles; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParser; @@ -247,6 +248,7 @@ private boolean prepareArtifact() throws IOException, InterruptedException { } private void activate(Path staged, boolean installed, String installedSha256) throws IOException { + DurableFiles.forceFile(staged); if (installed) { persistRollbackState(installedSha256); atomicMove(settings.jarFile(), settings.previousFile(), true); @@ -271,7 +273,7 @@ private void rollback() throws IOException { } private void persistRollbackState(String previousSha256) throws IOException { - Files.deleteIfExists(settings.healthCheckingFile()); + DurableFiles.deleteIfExists(settings.healthCheckingFile()); persistDigestState(settings.rollbackPendingFile(), previousSha256, settings.sha256(), "rollback"); } @@ -396,12 +398,12 @@ private String[] readDigestState(Path marker, String name) throws IOException { } private void clearPersistedRollbackState() throws IOException { - Files.deleteIfExists(settings.rollbackPendingFile()); - Files.deleteIfExists(settings.healthCheckingFile()); + DurableFiles.deleteIfExists(settings.rollbackPendingFile()); + DurableFiles.deleteIfExists(settings.healthCheckingFile()); } private void clearPersistedQuarantineState() throws IOException { - Files.deleteIfExists(settings.quarantineFile()); + DurableFiles.deleteIfExists(settings.quarantineFile()); } private LaunchedProcess launch(Path artifact) throws IOException { @@ -627,6 +629,7 @@ private static void atomicMove(Path source, Path target, boolean replace) throws } else { Files.move(source, target, StandardCopyOption.ATOMIC_MOVE); } + DurableFiles.forceMoveDirectories(source, target); } catch (AtomicMoveNotSupportedException e) { throw new IOException("Atomic Control artifact activation is unsupported", e); } @@ -888,7 +891,7 @@ public ManagedProcess launch(Settings settings, Path artifact, String launchId) } private static boolean isWindows() { - return System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win"); + return DurableFiles.isWindowsName(System.getProperty("os.name", "")); } } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java index 6999de42f9..73529cfbe6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyControlResultStore.java @@ -6,18 +6,17 @@ import java.nio.channels.FileChannel; import java.nio.channels.SeekableByteChannel; import java.nio.charset.StandardCharsets; -import java.nio.file.AccessDeniedException; import java.nio.file.Files; import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.util.LinkedHashMap; -import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.UUID; +import com.bencodez.votingplugin.util.DurableFiles; import com.google.gson.JsonArray; import com.google.gson.JsonElement; import com.google.gson.JsonObject; @@ -84,7 +83,7 @@ static void save(Path dataDirectory, Route route, Map result Path target = target(dataDirectory); if (results.isEmpty()) { if (Files.isSymbolicLink(target)) throw new IOException("Control proxy-result journal is unsafe"); - if (Files.deleteIfExists(target)) forceDirectory(dataDirectory); + DurableFiles.deleteIfExists(target); return; } if (results.size() > MAX_RESULTS) throw new IOException("Too many pending Control proxy results"); @@ -127,29 +126,12 @@ static void save(Path dataDirectory, Route route, Map result channel.force(true); } Files.move(staging, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); - forceDirectory(dataDirectory); + DurableFiles.forceDirectory(dataDirectory); } finally { Files.deleteIfExists(staging); } } - private static void forceDirectory(Path directory) throws IOException { - try { - try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { - channel.force(true); - } - } catch (AccessDeniedException unsupportedDirectoryHandle) { - // The Windows NIO provider cannot open directory handles. The staged file was - // already forced and the atomic move has completed, so do not turn every - // otherwise successful journal update into a permanent Windows failure. - if (!System.getProperty("os.name", "").toLowerCase(Locale.ROOT).contains("win")) { - throw unsupportedDirectoryHandle; - } - } catch (UnsupportedOperationException unsupportedDirectoryForce) { - // Some providers support atomic moves but expose no directory-force operation. - } - } - private static Path target(Path dataDirectory) { return dataDirectory.toAbsolutePath().normalize().resolve(FILE_NAME); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 5fb9fe8be9..2b6571faa1 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -22,6 +22,7 @@ import com.bencodez.simpleapi.file.velocity.VelocityYMLFile; import com.bencodez.votingplugin.proxy.VotingPluginProxyConfig; import com.bencodez.votingplugin.proxy.control.ProxyRoutingConfiguration; +import com.bencodez.votingplugin.util.DurableFiles; import org.spongepowered.configurate.ConfigurationNode; import org.spongepowered.configurate.yaml.YamlConfigurationLoader; @@ -143,7 +144,9 @@ public synchronized void loadControlConfiguration() throws IOException { private static void atomicReplace(Path source, Path target) throws IOException { try { + DurableFiles.forceFile(source); Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + DurableFiles.forceMoveDirectories(source, target); } catch (java.nio.file.AtomicMoveNotSupportedException e) { throw new IOException("Atomic Control configuration activation is unsupported", e); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java new file mode 100644 index 0000000000..90f6f18c1f --- /dev/null +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java @@ -0,0 +1,53 @@ +package com.bencodez.votingplugin.util; + +import java.io.IOException; +import java.nio.channels.FileChannel; +import java.nio.file.AccessDeniedException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.Locale; + +/** Cross-platform helpers for forcing file contents and published directory entries. */ +public final class DurableFiles { + private DurableFiles() { } + + public static void forceFile(Path file) throws IOException { + try (FileChannel channel = FileChannel.open(file, StandardOpenOption.WRITE, LinkOption.NOFOLLOW_LINKS)) { + channel.force(true); + } + } + + public static void forceDirectory(Path directory) throws IOException { + if (directory == null) return; + try { + try (FileChannel channel = FileChannel.open(directory, StandardOpenOption.READ)) { + channel.force(true); + } + } catch (AccessDeniedException unsupportedDirectoryHandle) { + // The Windows NIO provider cannot open directory handles. File contents are + // still forced before atomic publication; do not make persistence unusable. + if (!isWindowsName(System.getProperty("os.name", ""))) throw unsupportedDirectoryHandle; + } catch (UnsupportedOperationException unsupportedDirectoryForce) { + // Some providers support atomic moves but expose no directory-force operation. + } + } + + public static boolean deleteIfExists(Path target) throws IOException { + boolean deleted = Files.deleteIfExists(target); + if (deleted) forceDirectory(target.toAbsolutePath().normalize().getParent()); + return deleted; + } + + public static void forceMoveDirectories(Path source, Path target) throws IOException { + Path sourceParent = source.toAbsolutePath().normalize().getParent(); + Path targetParent = target.toAbsolutePath().normalize().getParent(); + forceDirectory(targetParent); + if (sourceParent != null && !sourceParent.equals(targetParent)) forceDirectory(sourceParent); + } + + public static boolean isWindowsName(String name) { + return name != null && name.trim().toLowerCase(Locale.ROOT).startsWith("windows"); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java index de7a4ccb97..fc97bb8347 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/backendproxy/transport/RedisBackendProxyTransportTest.java @@ -50,4 +50,19 @@ void legacyHandoffBufferDegradesAtItsFixedLimit() throws Exception { verify(messages, times(1)).onMessage(envelope); } + + @Test + void legacyHandoffBufferDegradesForAnOversizedEnvelope() throws Exception { + RedisBackendProxyTransport transport = new RedisBackendProxyTransport(null, new ProcessedVoteCache()); + GlobalMessageHandler messages = mock(GlobalMessageHandler.class); + Field handler = RedisBackendProxyTransport.class.getDeclaredField("messageHandler"); + handler.setAccessible(true); + handler.set(transport, messages); + JsonEnvelope envelope = JsonEnvelope.builder(VotingPluginWire.SUB_VOTE_UPDATE) + .put("payload", "x".repeat(ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES + 1)).build(); + + transport.dispatchLegacy(envelope); + + verify(messages).onMessage(envelope); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java index a7e23b9be8..66bcd22dfc 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/backendproxy/cache/ProcessedVoteCacheTest.java @@ -83,4 +83,19 @@ public void legacyRedisHandoffAccountingIsBounded() throws Exception { deliveries.setAccessible(true); assertEquals(4096, ((Map) deliveries.get(cache)).size()); } + + @Test + public void legacyRedisHandoffAccountingRejectsOversizedSignatures() throws Exception { + ProcessedVoteCache cache = new ProcessedVoteCache(); + Object previous = new Object(); + cache.registerRedisSubscriber(previous); + cache.registerRedisSubscriber(new Object()); + + assertTrue(cache.reserveLegacyRedisDelivery(previous, + "x".repeat(ProcessedVoteCache.MAX_LEGACY_REDIS_DELIVERY_BYTES + 1))); + + Field deliveries = ProcessedVoteCache.class.getDeclaredField("legacyRedisDeliveries"); + deliveries.setAccessible(true); + assertTrue(((Map) deliveries.get(cache)).isEmpty()); + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/util/DurableFilesTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/util/DurableFilesTest.java new file mode 100644 index 0000000000..c58b44a41a --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/util/DurableFilesTest.java @@ -0,0 +1,16 @@ +package com.bencodez.votingplugin.util; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class DurableFilesTest { + @Test + void identifiesWindowsWithoutMistakingDarwin() { + assertTrue(DurableFiles.isWindowsName("Windows 11")); + assertTrue(DurableFiles.isWindowsName("Windows Server 2022")); + assertFalse(DurableFiles.isWindowsName("Darwin")); + assertFalse(DurableFiles.isWindowsName("Linux")); + } +} From 34da3d0b9455d8fe30e659a44a7c28873bdf6947 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 21:49:06 -0600 Subject: [PATCH 72/73] Rollback configurations after post-rename sync failures --- .../control/BackendConfigurationService.java | 9 +++++-- .../proxy/bungee/BungeeConfig.java | 7 +++++- .../ProxyRoutingConfigurationService.java | 25 ++++++++++++------- .../proxy/velocity/VelocityConfig.java | 7 +++++- .../votingplugin/util/DurableFiles.java | 20 ++++++++++++--- .../ProxyRoutingConfigurationServiceTest.java | 25 +++++++++++++++++++ 6 files changed, 76 insertions(+), 17 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index dbee823003..45e4be0ebd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -84,8 +84,13 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); move(backupStaging, backup); if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); - move(staging, target); - installed = true; + try { + move(staging, target); + installed = true; + } catch (DurableFiles.PublishedException published) { + installed = true; + throw published; + } reload.run(fileName); String applied = readRaw(target, false); String installedRevision = revision(preview.resolvedContent()); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index 5e0ac585d6..f209c7e0ea 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -560,8 +560,13 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } - atomicReplace(stage, target); controlInstalledSnapshot = installedSnapshot; + try { + atomicReplace(stage, target); + } catch (IOException failure) { + if (!(failure instanceof DurableFiles.PublishedException)) controlInstalledSnapshot = null; + throw failure; + } data = latest; } finally { Files.deleteIfExists(stage); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java index 5ac9d270c0..572d7d4530 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java @@ -5,6 +5,7 @@ import java.util.Set; import com.bencodez.votingplugin.proxy.VotingPluginProxy; +import com.bencodez.votingplugin.util.DurableFiles; /** Validates, persists, reloads, and rolls back the bounded Control configuration domain. */ public final class ProxyRoutingConfigurationService { @@ -61,6 +62,8 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t platform.persist(proposal, expectedRevision); } catch (com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException e) { throw new StaleRevisionException(); + } catch (DurableFiles.PublishedException published) { + throw rollbackAfterFailure(published); } try { platform.reload(); @@ -73,16 +76,20 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t } catch (StaleRevisionException stale) { throw stale; } catch (Exception e) { - boolean rolledBack = false; - try { - platform.rollback(); - platform.reload(); - rolledBack = true; - } catch (Exception rollbackFailure) { - e.addSuppressed(rollbackFailure); - } - throw new ApplyFailureException(rolledBack, e); + throw rollbackAfterFailure(e); + } + } + + private ApplyFailureException rollbackAfterFailure(Exception failure) { + boolean rolledBack = false; + try { + platform.rollback(); + platform.reload(); + rolledBack = true; + } catch (Exception rollbackFailure) { + failure.addSuppressed(rollbackFailure); } + return new ApplyFailureException(rolledBack, failure); } private void reconcileConcurrentEdit() throws Exception { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index 2b6571faa1..a4c810e62b 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -74,8 +74,13 @@ public synchronized void persistControlProxyRouting(boolean sendVotesToAllServer if (!java.util.Arrays.equals(sourceSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } - atomicReplace(stage, target); controlInstalledSnapshot = installedSnapshot; + try { + atomicReplace(stage, target); + } catch (IOException failure) { + if (!(failure instanceof DurableFiles.PublishedException)) controlInstalledSnapshot = null; + throw failure; + } } finally { Files.deleteIfExists(stage); if (backupStage != null) Files.deleteIfExists(backupStage); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java index 90f6f18c1f..b57fdb72fd 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/DurableFiles.java @@ -41,13 +41,25 @@ public static boolean deleteIfExists(Path target) throws IOException { } public static void forceMoveDirectories(Path source, Path target) throws IOException { - Path sourceParent = source.toAbsolutePath().normalize().getParent(); - Path targetParent = target.toAbsolutePath().normalize().getParent(); - forceDirectory(targetParent); - if (sourceParent != null && !sourceParent.equals(targetParent)) forceDirectory(sourceParent); + try { + Path sourceParent = source.toAbsolutePath().normalize().getParent(); + Path targetParent = target.toAbsolutePath().normalize().getParent(); + forceDirectory(targetParent); + if (sourceParent != null && !sourceParent.equals(targetParent)) forceDirectory(sourceParent); + } catch (IOException failure) { + throw new PublishedException(failure); + } } public static boolean isWindowsName(String name) { return name != null && name.trim().toLowerCase(Locale.ROOT).startsWith("windows"); } + + /** Indicates that an atomic rename completed before metadata writeback failed. */ + @SuppressWarnings("serial") + public static final class PublishedException extends IOException { + public PublishedException(IOException cause) { + super("File was published but its directory metadata could not be forced", cause); + } + } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java index ef250979ac..ed413ac0fd 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java @@ -12,6 +12,8 @@ import org.junit.jupiter.api.Test; +import com.bencodez.votingplugin.util.DurableFiles; + class ProxyRoutingConfigurationServiceTest { @Test void validatesBlockedServersAgainstCompleteConfiguredSet() { @@ -54,6 +56,24 @@ void persistenceRevisionRaceRemainsAStaleRevision() { assertTrue(platform.reloadCalls == 0); } + @Test + void postRenameDirectoryForceFailureRollsBackPublishedConfiguration() { + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + FakePlatform platform = new FakePlatform(); + platform.current = current; + platform.failAfterPublication = true; + + ProxyRoutingConfigurationService.ApplyFailureException failure = assertThrows( + ProxyRoutingConfigurationService.ApplyFailureException.class, + () -> new ProxyRoutingConfigurationService(platform).apply( + new ProxyRoutingConfiguration(true, List.of()), current.revision())); + + assertTrue(failure.rolledBack()); + assertTrue(platform.rolledBack); + assertTrue(platform.current.equals(current)); + assertTrue(platform.reloadCalls == 1); + } + @Test void failedReloadPreservesAConcurrentEditDetectedByTheAdapter() { ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); @@ -105,21 +125,26 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ private boolean failFirstReload; private boolean rolledBack; private boolean staleDuringPersist; + private boolean failAfterPublication; private boolean manualEditDuringFailedReload; private boolean manualEditDuringSuccessfulReload; private int manualEditsDuringSuccessfulReloads; private ProxyRoutingConfiguration installed; + private ProxyRoutingConfiguration previous; private int reloadCalls; @Override public ProxyRoutingConfiguration read() { return current; } @Override public Set configuredServers() { return configuredServers; } @Override public void persist(ProxyRoutingConfiguration proposal, String expectedRevision) throws IOException { if (staleDuringPersist) throw new com.bencodez.votingplugin.proxy.VotingPluginProxyConfig.StaleControlRevisionException(); + previous = current; current = proposal; installed = proposal; + if (failAfterPublication) throw new DurableFiles.PublishedException(new IOException("directory force failed")); } @Override public void rollback() throws IOException { if (!current.equals(installed)) throw new IOException("active configuration changed"); + current = previous; rolledBack = true; } @Override public void reload() throws Exception { From aed406df1926681c0728c835ca3dd69b6b0d1c33 Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 27 Aug 2026 22:01:18 -0600 Subject: [PATCH 73/73] Reload configurations after published rollback --- .../control/BackendConfigurationService.java | 23 +++++++++++--- .../proxy/bungee/BungeeConfig.java | 7 ++++- .../ProxyRoutingConfigurationService.java | 10 +++++- .../proxy/velocity/VelocityConfig.java | 7 ++++- .../BackendConfigurationServiceTest.java | 31 +++++++++++++++++++ .../ProxyRoutingConfigurationServiceTest.java | 25 +++++++++++++++ 6 files changed, 95 insertions(+), 8 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java index 45e4be0ebd..b43256a21c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/control/BackendConfigurationService.java @@ -38,14 +38,20 @@ public final class BackendConfigurationService { private final Path dataDirectory; private final ApplyAction reload; + private final MoveAction mover; public BackendConfigurationService(Path dataDirectory, ReloadAction reload) { this(dataDirectory, ignored -> reload.run()); } public BackendConfigurationService(Path dataDirectory, ApplyAction reload) { + this(dataDirectory, reload, BackendConfigurationService::move); + } + + BackendConfigurationService(Path dataDirectory, ApplyAction reload, MoveAction mover) { this.dataDirectory = dataDirectory.toAbsolutePath().normalize(); this.reload = reload; + this.mover = mover; } public Document read(String fileName) throws IOException { @@ -82,10 +88,10 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte Files.writeString(backupStaging, current, StandardCharsets.UTF_8, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); - move(backupStaging, backup); + mover.move(backupStaging, backup); if (!revision(readRaw(target, false)).equals(expectedRevision)) throw new StaleRevisionException(); try { - move(staging, target); + mover.move(staging, target); installed = true; } catch (DurableFiles.PublishedException published) { installed = true; @@ -110,7 +116,13 @@ public ApplyResult apply(String fileName, String proposedContent, String expecte if (!revision(readRaw(target, false)).equals(installedRevision)) { throw new IOException("Managed configuration changed while reload failed; backup was not restored"); } - copyBackupNoFollow(backup, target, installedRevision); + try { + copyBackupNoFollow(backup, target, installedRevision); + } catch (DurableFiles.PublishedException published) { + // The backup is already active. Preserve the durability warning, but + // still reload it so runtime and disk cannot diverge. + failure.addSuppressed(published); + } reload.run(fileName); rolledBack = true; } catch (Exception rollbackFailure) { @@ -137,7 +149,7 @@ private static void rejectSymbolicBackup(Path backup) throws IOException { if (Files.isSymbolicLink(backup)) throw new IOException("Symbolic configuration backups are not allowed"); } - private static void copyBackupNoFollow(Path backup, Path target, String expectedTargetRevision) throws IOException { + private void copyBackupNoFollow(Path backup, Path target, String expectedTargetRevision) throws IOException { if (!Files.isRegularFile(backup, LinkOption.NOFOLLOW_LINKS)) { throw new IOException("Control backup is unavailable or unsafe"); } @@ -150,7 +162,7 @@ private static void copyBackupNoFollow(Path backup, Path target, String expected if (!revision(readRaw(target, false)).equals(expectedTargetRevision)) { throw new IOException("Managed configuration changed while rollback was staged"); } - move(staging, target); + mover.move(staging, target); } finally { Files.deleteIfExists(staging); } @@ -432,6 +444,7 @@ public record QuickPreview(QuickProposal proposal, String revision, List @FunctionalInterface public interface ReloadAction { void run() throws Exception; } @FunctionalInterface public interface ApplyAction { void run(String fileName) throws Exception; } + @FunctionalInterface interface MoveAction { void move(Path source, Path target) throws IOException; } @SuppressWarnings("serial") public static final class StaleRevisionException extends RuntimeException { } @SuppressWarnings("serial") public static final class ApplyFailureException extends IOException { private final boolean rolledBack; diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java index f209c7e0ea..2022295379 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/bungee/BungeeConfig.java @@ -599,7 +599,12 @@ public synchronized void rollbackControlProxyRouting() throws IOException { if (!java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } - atomicReplace(stage, target); + try { + atomicReplace(stage, target); + } catch (DurableFiles.PublishedException published) { + controlInstalledSnapshot = null; + throw published; + } } finally { Files.deleteIfExists(stage); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java index 572d7d4530..2c10ade56f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationService.java @@ -82,13 +82,21 @@ public void apply(ProxyRoutingConfiguration proposal, String expectedRevision) t private ApplyFailureException rollbackAfterFailure(Exception failure) { boolean rolledBack = false; + DurableFiles.PublishedException publicationFailure = null; try { - platform.rollback(); + try { + platform.rollback(); + } catch (DurableFiles.PublishedException published) { + // The restored file is already active. Continue the reload so runtime + // state follows disk, while retaining the durability warning. + publicationFailure = published; + } platform.reload(); rolledBack = true; } catch (Exception rollbackFailure) { failure.addSuppressed(rollbackFailure); } + if (publicationFailure != null) failure.addSuppressed(publicationFailure); return new ApplyFailureException(rolledBack, failure); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java index a4c810e62b..2e167420de 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/proxy/velocity/VelocityConfig.java @@ -112,7 +112,12 @@ public synchronized void rollbackControlProxyRouting() throws IOException { if (!java.util.Arrays.equals(controlInstalledSnapshot, Files.readAllBytes(target))) { throw new StaleControlRevisionException(); } - atomicReplace(stage, target); + try { + atomicReplace(stage, target); + } catch (DurableFiles.PublishedException published) { + controlInstalledSnapshot = null; + throw published; + } } finally { Files.deleteIfExists(stage); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java index b0cf6d2ae1..80871f58fe 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/control/BackendConfigurationServiceTest.java @@ -5,8 +5,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; @@ -14,6 +16,8 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; +import com.bencodez.votingplugin.util.DurableFiles; + class BackendConfigurationServiceTest { @TempDir Path directory; @@ -101,6 +105,33 @@ class BackendConfigurationServiceTest { assertTrue(service.read("BungeeSettings.yml").content().contains("PLUGINMESSAGING")); } + @Test void publishedRollbackStillReloadsTheRestoredConfiguration() throws Exception { + Path config = directory.resolve("Config.yml"); + Files.writeString(config, "Feature: before\n"); + AtomicInteger reloads = new AtomicInteger(); + AtomicInteger moves = new AtomicInteger(); + BackendConfigurationService service = new BackendConfigurationService(directory, + (BackendConfigurationService.ApplyAction) fileName -> { + if (reloads.incrementAndGet() == 1) throw new IOException("reload failed"); + }, (source, target) -> { + Files.move(source, target, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + if (moves.incrementAndGet() == 3) { + throw new DurableFiles.PublishedException(new IOException("directory force failed")); + } + }); + BackendConfigurationService.Document before = service.read("Config.yml"); + + BackendConfigurationService.ApplyFailureException failure = assertThrows( + BackendConfigurationService.ApplyFailureException.class, + () -> service.apply("Config.yml", "Feature: proposed\n", before.revision())); + + assertTrue(failure.rolledBack()); + assertEquals(2, reloads.get()); + assertEquals("Feature: before\n", Files.readString(config)); + assertTrue(java.util.Arrays.stream(failure.getCause().getSuppressed()) + .anyMatch(DurableFiles.PublishedException.class::isInstance)); + } + @Test void failedReloadDoesNotOverwriteAConcurrentManualEdit() throws Exception { Path config = directory.resolve("Config.yml"); Files.writeString(config, "Feature: before\n"); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java index ed413ac0fd..ff7df8876f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/proxy/control/ProxyRoutingConfigurationServiceTest.java @@ -74,6 +74,27 @@ void postRenameDirectoryForceFailureRollsBackPublishedConfiguration() { assertTrue(platform.reloadCalls == 1); } + @Test + void publishedRollbackStillReloadsTheRestoredConfiguration() { + ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); + FakePlatform platform = new FakePlatform(); + platform.current = current; + platform.failFirstReload = true; + platform.failAfterRollbackPublication = true; + + ProxyRoutingConfigurationService.ApplyFailureException failure = assertThrows( + ProxyRoutingConfigurationService.ApplyFailureException.class, + () -> new ProxyRoutingConfigurationService(platform).apply( + new ProxyRoutingConfiguration(true, List.of()), current.revision())); + + assertTrue(failure.rolledBack()); + assertTrue(platform.rolledBack); + assertTrue(platform.current.equals(current)); + assertTrue(platform.reloadCalls == 2); + assertTrue(java.util.Arrays.stream(failure.getCause().getSuppressed()) + .anyMatch(DurableFiles.PublishedException.class::isInstance)); + } + @Test void failedReloadPreservesAConcurrentEditDetectedByTheAdapter() { ProxyRoutingConfiguration current = new ProxyRoutingConfiguration(false, List.of()); @@ -126,6 +147,7 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ private boolean rolledBack; private boolean staleDuringPersist; private boolean failAfterPublication; + private boolean failAfterRollbackPublication; private boolean manualEditDuringFailedReload; private boolean manualEditDuringSuccessfulReload; private int manualEditsDuringSuccessfulReloads; @@ -146,6 +168,9 @@ private static final class FakePlatform implements ProxyRoutingConfigurationServ if (!current.equals(installed)) throw new IOException("active configuration changed"); current = previous; rolledBack = true; + if (failAfterRollbackPublication) { + throw new DurableFiles.PublishedException(new IOException("directory force failed")); + } } @Override public void reload() throws Exception { reloadCalls++;